TrafficWheel/model/EXP/trash/EXP27.py

import torch
import torch.nn as nn
import torch.nn.functional as F

"""
(无效)节点混合专家
"""

class MANBA_Block(nn.Module):
    def __init__(self, input_dim, hidden_dim):
        super().__init__()
        self.attn = nn.MultiheadAttention(embed_dim=input_dim, num_heads=4, batch_first=True)
        self.ffn = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, input_dim)
        )
        self.norm1 = nn.LayerNorm(input_dim)
        self.norm2 = nn.LayerNorm(input_dim)

    def forward(self, x):
        # x: (B, N, input_dim)
        res = x
        x_attn, _ = self.attn(x, x, x)
        x = self.norm1(res + x_attn)
        res2 = x
        x_ffn = self.ffn(x)
        x = self.norm2(res2 + x_ffn)
        return x


class ExpertBlock(nn.Module):
    """
    Mixture-of-Experts block: routes each node's representation to a selected expert or a shared expert.
    """
    def __init__(self, hidden_dim, num_experts):
        super().__init__()
        self.num_experts = num_experts
        # gating network projects to num_experts + 1 (extra shared expert)
        self.gate = nn.Linear(hidden_dim, num_experts + 1)
        # per-expert FFNs
        self.experts = nn.ModuleList([
            nn.Sequential(
                nn.Linear(hidden_dim, hidden_dim * 2),
                nn.ReLU(),
                nn.Linear(hidden_dim * 2, hidden_dim)
            ) for _ in range(num_experts)
        ])
        # shared expert
        self.shared_expert = nn.Sequential(
            nn.Linear(hidden_dim, hidden_dim * 2),
            nn.ReLU(),
            nn.Linear(hidden_dim * 2, hidden_dim)
        )

    def forward(self, x):
        # x: (B, N, hidden_dim)
        B, N, D = x.shape
        # flatten to (B*N, D)
        flat = x.view(B * N, D)
        # compute gating scores and select expert per node
        scores = F.softmax(self.gate(flat), dim=-1)  # (B*N, num_experts+1)
        idx = scores.argmax(dim=-1)                  # (B*N,)

        out_flat = torch.zeros_like(flat)
        # apply each expert
        for e in range(self.num_experts):
            mask = (idx == e)
            if mask.any():
                out_flat[mask] = self.experts[e](flat[mask])
        # apply shared expert for last index
        shared_mask = (idx == self.num_experts)
        if shared_mask.any():
            out_flat[shared_mask] = self.shared_expert(flat[shared_mask])

        # reshape back to (B, N, D)
        return out_flat.view(B, N, D)


class MLP(nn.Module):
    def __init__(self, in_dim, hidden_dims, out_dim, activation=nn.ReLU):
        super().__init__()
        dims = [in_dim] + hidden_dims + [out_dim]
        layers = []
        for i in range(len(dims) - 2):
            layers += [nn.Linear(dims[i], dims[i+1]), activation()]
        layers += [nn.Linear(dims[-2], dims[-1])]
        self.net = nn.Sequential(*layers)

    def forward(self, x):
        return self.net(x)


class SandwichBlock(nn.Module):
    def __init__(self, num_nodes, embed_dim, hidden_dim, num_experts):
        super().__init__()
        self.manba1 = MANBA_Block(hidden_dim, hidden_dim * 2)
        self.expert_block = ExpertBlock(hidden_dim, num_experts)
        self.manba2 = MANBA_Block(hidden_dim, hidden_dim * 2)

    def forward(self, h):
        h1 = self.manba1(h)
        h2 = self.expert_block(h1)
        h3 = self.manba2(h2)
        return h3


class EXP(nn.Module):
    def __init__(self, args):
        super().__init__()
        self.horizon     = args['horizon']
        self.output_dim  = args['output_dim']
        self.seq_len     = args.get('in_len', 12)
        self.hidden_dim  = args.get('hidden_dim', 64)
        self.num_nodes   = args['num_nodes']
        self.embed_dim   = args.get('embed_dim', 16)
        self.num_experts = args.get('num_experts', 8)  # number of private experts

        # discrete time embeddings
        self.time_slots    = args.get('time_slots', 24 * 60 // args.get('time_slot', 5))
        self.time_embedding = nn.Embedding(self.time_slots, self.hidden_dim)
        self.day_embedding  = nn.Embedding(7, self.hidden_dim)

        # input projection
        self.input_proj = MLP(
            in_dim      = self.seq_len,
            hidden_dims = [self.hidden_dim],
            out_dim     = self.hidden_dim
        )

        # two Sandwich blocks with MoE
        self.sandwich1 = SandwichBlock(self.num_nodes, self.embed_dim, self.hidden_dim, self.num_experts)
        self.sandwich2 = SandwichBlock(self.num_nodes, self.embed_dim, self.hidden_dim, self.num_experts)

        # output projection
        self.out_proj = MLP(
            in_dim      = self.hidden_dim,
            hidden_dims = [2 * self.hidden_dim],
            out_dim     = self.horizon * self.output_dim
        )

    def forward(self, x):
        """
        x: (B, T, N, D_total)
           x[...,0]= flow, x[...,1]=time_in_day, x[...,2]=day_in_week
        """
        x_flow = x[..., 0]
        x_time = x[..., 1]
        x_day  = x[..., 2]

        B, T, N = x_flow.shape
        assert T == self.seq_len

        # project flow history
        x_flat = x_flow.permute(0, 2, 1).reshape(B * N, T)
        h0 = self.input_proj(x_flat).view(B, N, self.hidden_dim)

        # time & day embeddings at last step
        t_idx = (x_time[:, -1, :,] * (self.time_slots - 1)).long()
        d_idx = x_day[:,  -1, :,].long()
        time_emb = self.time_embedding(t_idx)
        day_emb  = self.day_embedding(d_idx)
        h0 = h0 + time_emb + day_emb

        # two MoE Sandwich blocks + residuals
        h1 = self.sandwich1(h0) + h0
        h2 = self.sandwich2(h1) + h1

        # output
        out = self.out_proj(h2)
        out = out.view(B, N, self.horizon, self.output_dim)
        out = out.permute(0, 2, 1, 3)
        return out