Project-I/models/STGODE_LLM/STGODE_LLM.py

import torch
import math
import torch.nn as nn
import torch.nn.functional as F

from models.STGODE.odegcn import ODEG
from models.STGODE.adj import get_A_hat


class Chomp1d(nn.Module):
    def __init__(self, chomp_size):
        super(Chomp1d, self).__init__()
        self.chomp_size = chomp_size

    def forward(self, x):
        return x[:, :, :, :-self.chomp_size].contiguous()


class TemporalConvNet(nn.Module):
    def __init__(self, num_inputs, num_channels, kernel_size=2, dropout=0.2):
        super(TemporalConvNet, self).__init__()
        layers = []
        num_levels = len(num_channels)
        for i in range(num_levels):
            dilation_size = 2 ** i
            in_channels = num_inputs if i == 0 else num_channels[i - 1]
            out_channels = num_channels[i]
            padding = (kernel_size - 1) * dilation_size
            self.conv = nn.Conv2d(in_channels, out_channels, (1, kernel_size), dilation=(1, dilation_size),
                                  padding=(0, padding))
            self.conv.weight.data.normal_(0, 0.01)
            self.chomp = Chomp1d(padding)
            self.relu = nn.ReLU()
            self.dropout = nn.Dropout(dropout)
            layers += [nn.Sequential(self.conv, self.chomp, self.relu, self.dropout)]

        self.network = nn.Sequential(*layers)
        self.downsample = nn.Conv2d(num_inputs, num_channels[-1], (1, 1)) if num_inputs != num_channels[-1] else None
        if self.downsample:
            self.downsample.weight.data.normal_(0, 0.01)

    def forward(self, x):
        y = x.permute(0, 3, 1, 2)
        y = F.relu(self.network(y) + self.downsample(y) if self.downsample else y)
        y = y.permute(0, 2, 3, 1)
        return y


class STGCNBlock(nn.Module):
    def __init__(self, in_channels, out_channels, num_nodes, A_hat):
        super(STGCNBlock, self).__init__()
        self.A_hat = A_hat
        self.temporal1 = TemporalConvNet(num_inputs=in_channels, num_channels=out_channels)
        self.odeg = ODEG(out_channels[-1], 12, A_hat, time=6)
        self.temporal2 = TemporalConvNet(num_inputs=out_channels[-1], num_channels=out_channels)
        self.batch_norm = nn.BatchNorm2d(num_nodes)

    def forward(self, X):
        t = self.temporal1(X)
        t = self.odeg(t)
        t = self.temporal2(F.relu(t))
        return self.batch_norm(t)


class GPT2Backbone(nn.Module):
    def __init__(self, hidden_size: int, n_layer: int = 4, n_head: int = 4, n_embd: int | None = None, use_pretrained: bool = True):
        super().__init__()
        self.hidden_size = hidden_size
        self.use_transformers = False
        self.model = None
        if n_embd is None:
            n_embd = hidden_size
        if use_pretrained:
            try:
                from transformers import GPT2Model, GPT2Config
                config = GPT2Config(n_embd=n_embd, n_layer=n_layer, n_head=n_head, n_positions=1024, n_ctx=1024, vocab_size=1)
                self.model = GPT2Model(config)
                self.use_transformers = True
            except Exception:
                self.use_transformers = False
        if not self.use_transformers:
            encoder_layer = nn.TransformerEncoderLayer(d_model=hidden_size, nhead=n_head, batch_first=True)
            self.model = nn.TransformerEncoder(encoder_layer, num_layers=n_layer)

    def forward(self, inputs_embeds: torch.Tensor) -> torch.Tensor:
        if self.use_transformers:
            outputs = self.model(inputs_embeds=inputs_embeds)
            return outputs.last_hidden_state
        else:
            return self.model(inputs_embeds)


class ODEGCN_LLM(nn.Module):
    def __init__(self, config):
        super(ODEGCN_LLM, self).__init__()
        args = config['model']
        num_nodes = config['data']['num_nodes']
        num_features = args['num_features']
        num_timesteps_input = args['history']
        num_timesteps_output = args['horizon']
        A_sp_hat, A_se_hat = get_A_hat(config)

        self.sp_blocks = nn.ModuleList(
            [nn.Sequential(
                STGCNBlock(in_channels=num_features, out_channels=[64, 32, 64], num_nodes=num_nodes, A_hat=A_sp_hat),
                STGCNBlock(in_channels=64, out_channels=[64, 32, 64], num_nodes=num_nodes, A_hat=A_sp_hat)) for _ in range(3)
            ])

        self.se_blocks = nn.ModuleList(
            [nn.Sequential(
                STGCNBlock(in_channels=num_features, out_channels=[64, 32, 64], num_nodes=num_nodes, A_hat=A_se_hat),
                STGCNBlock(in_channels=64, out_channels=[64, 32, 64], num_nodes=num_nodes, A_hat=A_se_hat)) for _ in range(3)
            ])

        self.history = num_timesteps_input
        self.horizon = num_timesteps_output

        hidden_size = int(args.get('llm_hidden', 128))
        llm_layers = int(args.get('llm_layers', 4))
        llm_heads = int(args.get('llm_heads', 4))
        use_pretrained = bool(args.get('llm_pretrained', True))

        self.to_llm_embed = nn.Linear(64, hidden_size)
        self.gpt2 = GPT2Backbone(hidden_size=hidden_size, n_layer=llm_layers, n_head=llm_heads, use_pretrained=use_pretrained)
        self.proj_head = nn.Sequential(
            nn.Linear(hidden_size, hidden_size),
            nn.ReLU(),
            nn.Linear(hidden_size, self.horizon)
        )

    def forward(self, x):
        x = x[..., 0:1].permute(0, 2, 1, 3)
        outs = []
        for blk in self.sp_blocks:
            outs.append(blk(x))
        for blk in self.se_blocks:
            outs.append(blk(x))
        outs = torch.stack(outs)
        x = torch.max(outs, dim=0)[0]

        # x: (B, N, T, 64) physical quantities after ODE-based transform
        B, N, T, C = x.shape
        x = self.to_llm_embed(x)  # (B, N, T, H)
        x = x.permute(0, 1, 2, 3).contiguous().view(B * N, T, -1)  # (B*N, T, H)

        llm_hidden = self.gpt2(inputs_embeds=x)  # (B*N, T, H)
        last_state = llm_hidden[:, -1, :]  # (B*N, H)
        y = self.proj_head(last_state)  # (B*N, horizon)
        y = y.view(B, N, self.horizon).permute(0, 2, 1).unsqueeze(-1)  # (B, horizon, N, 1)
        return y