111

algorithms/algorithm/r_actor_critic.py

@@ -9,7 +9,7 @@ import torch
 import torch.nn as nn
 from algorithms.utils.util import init, check
 from algorithms.utils.cnn import CNNBase
-from algorithms.utils.mlp import MLPBase
+from algorithms.utils.mlp import MLPBase, MLPBaseWithTrans, MLPBaseGPT2
 from algorithms.utils.rnn import RNNLayer
 from algorithms.utils.act import ACTLayer
 from algorithms.utils.popart import PopArt
@@ -37,7 +37,7 @@ class R_Actor(nn.Module):
         self.tpdv = dict(dtype=torch.float32, device=device)
 
         obs_shape = get_shape_from_obs_space(obs_space)
-        base = CNNBase if len(obs_shape) == 3 else MLPBase
+        base = CNNBase if len(obs_shape) == 3 else MLPBaseGPT2
         self.base = base(args, obs_shape)
 
         if self._use_naive_recurrent_policy or self._use_recurrent_policy:
@@ -134,7 +134,9 @@ class R_Critic(nn.Module):
         init_method = [nn.init.xavier_uniform_, nn.init.orthogonal_][self._use_orthogonal]
 
         cent_obs_shape = get_shape_from_obs_space(cent_obs_space)
-        base = CNNBase if len(cent_obs_shape) == 3 else MLPBase
+        base = CNNBase if len(cent_obs_shape) == 3 else MLPBaseGPT2
+        print("cent_obs_shape:" )
+        print(cent_obs_shape)
         self.base = base(args, cent_obs_shape)
 
         if self._use_naive_recurrent_policy or self._use_recurrent_policy:

algorithms/utils/act.py

@@ -15,9 +15,13 @@ class ACTLayer(nn.Module):
         self.mixed_action = False
         self.multi_discrete = False
         self.continuous_action = False
+        self.tanh = nn.Tanh()
+
+        print(action_space)
 
         if action_space.__class__.__name__ == "Discrete":
             action_dim = action_space.n
+            print(action_dim)
             self.action_out = Categorical(inputs_dim, action_dim, use_orthogonal, gain)
         elif action_space.__class__.__name__ == "Box":
             self.continuous_action = True
@@ -30,10 +34,12 @@ class ACTLayer(nn.Module):
             self.multi_discrete = True
             action_dims = action_space.high - action_space.low + 1
             self.action_outs = []
+            print(action_dims)
             for action_dim in action_dims:
                 self.action_outs.append(Categorical(inputs_dim, action_dim, use_orthogonal, gain))
             self.action_outs = nn.ModuleList(self.action_outs)
         else:  # discrete + continous
+            print("mixed_action")
             self.mixed_action = True
             continous_dim = action_space[0].shape[0]
             discrete_dim = action_space[1].n
@@ -82,6 +88,8 @@ class ACTLayer(nn.Module):
             action_logit = self.action_out(x)
             actions = action_logit.mode() if deterministic else action_logit.sample()
             action_log_probs = action_logit.log_probs(actions)
+            #actions = torch.sigmoid(actions)
+            actions = self.tanh(actions)
             # actions.append(action.float())
             # action_log_probs.append(action_log_prob)
             # actions = torch.cat(actions, -1)

algorithms/utils/mlp.py

+import math
+
+import numpy
+import numpy as np
+import torch
 import torch.nn as nn
-from .util import init, get_clones
+from torchvision.models.video.mvit import PositionalEncoding
 
+from envs.env_core import EnvCore
+from .util import init, get_clones
+from transformers import GPT2Config, GPT2Model
 """MLP modules."""
 
+Feature = 4
+
 class MLPLayer(nn.Module):
     def __init__(self, input_dim, hidden_size, layer_N, use_orthogonal, use_ReLU):
         super(MLPLayer, self).__init__()
@@ -27,6 +37,28 @@ class MLPLayer(nn.Module):
             x = self.fc2[i](x)
         return x
 
+class MLPLayer2(nn.Module):
+    def __init__(self, input_dim, hidden_size, layer_N, use_orthogonal, use_ReLU):
+        super(MLPLayer2, self).__init__()
+        self._layer_N = layer_N
+
+        active_func = [nn.Tanh(), nn.ReLU()][use_ReLU]
+        init_method = [nn.init.xavier_uniform_, nn.init.orthogonal_][use_orthogonal]
+        gain = nn.init.calculate_gain(['tanh', 'relu'][use_ReLU])
+
+        def init_(m):
+            return init(m, init_method, lambda x: nn.init.constant_(x, 0), gain=gain)
+
+        self.fc1 = nn.Sequential(
+            init_(nn.Linear(input_dim, hidden_size)), active_func, nn.LayerNorm(hidden_size))
+        self.fc2 = nn.Linear(hidden_size, Feature)
+
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.fc2(x)
+        return x
+
 
 class MLPBase(nn.Module):
     def __init__(self, args, obs_shape, cat_self=True, attn_internal=False):
@@ -47,10 +79,315 @@ class MLPBase(nn.Module):
         self.mlp = MLPLayer(obs_dim, self.hidden_size,
                               self._layer_N, self._use_orthogonal, self._use_ReLU)
 
+
     def forward(self, x):
         if self._use_feature_normalization:
             x = self.feature_norm(x)
 
         x = self.mlp(x)
 
-        return x
\ No newline at end of file
+        return x
+
+class MLPBaseGPT2(nn.Module):
+    def __init__(self, args, obs_shape, cat_self=True, attn_internal=False):
+        super(MLPBaseGPT2, self).__init__()
+
+        self._use_feature_normalization = args.use_feature_normalization
+        self._use_orthogonal = args.use_orthogonal
+        self._use_ReLU = args.use_ReLU
+        self._stacked_frames = args.stacked_frames
+        self._layer_N = args.layer_N
+        self.hidden_size = args.hidden_size
+
+        obs_dim = obs_shape[0]
+
+        if self._use_feature_normalization:
+            self.feature_norm = nn.LayerNorm(obs_dim)
+
+        config = GPT2Config(n_embd=128,
+                            n_layer=2,
+                            n_head=4)
+        self.mlp = new_mlp_gpt2(obs_dim,
+                    128,
+                    1,
+                    config = config)
+
+
+
+
+    def forward(self, x):
+        if self._use_feature_normalization:
+            x = self.feature_norm(x)
+
+        x = self.mlp(x)
+
+        return x
+
+
+class MLPBaseWithTrans(nn.Module):
+    def __init__(self, args, obs_shape, cat_self=True, attn_internal=False):
+        super(MLPBaseWithTrans, self).__init__()
+
+        self._use_feature_normalization = args.use_feature_normalization
+        self._use_orthogonal = args.use_orthogonal
+        self._use_ReLU = args.use_ReLU
+        self._stacked_frames = args.stacked_frames
+        self._layer_N = args.layer_N
+        self.hidden_size = args.hidden_size
+
+        obs_dim = obs_shape[0]
+
+        if self._use_feature_normalization:
+            self.feature_norm = nn.LayerNorm(obs_dim)
+
+        self.mlp = MLPLayer((EnvCore.AerialVehiclesNum+3) * Feature, self.hidden_size,
+                              self._layer_N, self._use_orthogonal, self._use_ReLU)
+
+        self.transLayer = TransLayer(args, obs_shape)
+
+    def forward(self, x):
+        if self._use_feature_normalization:
+            x = self.feature_norm(x)
+
+        x = self.transLayer(x)
+
+        x = x[0]
+        res = []
+        for i in  range (0, len(x)):
+            for j in  range (0, Feature):
+                res.append(x[i][j])
+        w = []
+        w.append(res)
+        w = torch.Tensor(w)
+
+
+        x = self.mlp(w)
+
+        return x
+
+
+
+class TransLayer(nn.Module):
+    def __init__(self, args, obs_shape, cat_self=True, attn_internal=False):
+        super(TransLayer, self).__init__()
+
+        self._use_feature_normalization = args.use_feature_normalization
+        self._use_orthogonal = args.use_orthogonal
+        self._use_ReLU = args.use_ReLU
+        self._stacked_frames = args.stacked_frames
+        self._layer_N = args.layer_N
+        self.hidden_size = args.hidden_size
+
+        obs_dim = obs_shape[0]
+
+        if self._use_feature_normalization:
+            self.feature_norm = nn.LayerNorm(obs_dim)
+
+        self.uavLayer = MLPLayer2(EnvCore.dimension1, self.hidden_size,
+                                  self._layer_N, self._use_orthogonal, self._use_ReLU)
+        self.cacheLayer = MLPLayer2(EnvCore.dimension2, self.hidden_size,
+                                  self._layer_N, self._use_orthogonal, self._use_ReLU)
+        self.relationLayer = MLPLayer2(EnvCore.dimension3, self.hidden_size,
+                                  self._layer_N, self._use_orthogonal, self._use_ReLU)
+        self.kLayer = MLPLayer2(EnvCore.dimension4, self.hidden_size,
+                                  self._layer_N, self._use_orthogonal, self._use_ReLU)
+
+        self.attentionLayer = MultiHeadAttention(Feature, 6, 64, 64)
+
+    def forward(self, x):
+
+        length = EnvCore.AerialVehiclesNum + 1
+        start = 0
+        result = []
+
+
+        for i in range(0, EnvCore.AerialVehiclesNum):
+            w = x[0][start:start + length]
+            start = start + length
+            result.append(self.uavLayer(w))
+
+        length = EnvCore.dimension2
+        w = x[0][start:start + length]
+        start = start + length
+        result.append(self.cacheLayer(w))
+
+        length = EnvCore.dimension3
+        w = x[0][start:start + length]
+        start = start + length
+        result.append(self.relationLayer(w))
+
+        length = EnvCore.dimension4
+        w = x[0][start:start + length]
+        start = start + length
+        result.append(self.kLayer(w))
+        list_mask = []
+        list_mask.append(result)
+        list_mask = torch.tensor([item.detach().numpy() for item in result])
+        list_mask2 = []
+        list_mask2.append(list_mask)
+        list_mask = torch.tensor([item.detach().numpy() for item in list_mask2])
+
+        list_mask = torch.Tensor(list_mask)
+
+        mask = get_attn_pad_mask(list_mask, list_mask)
+
+
+        list_attn2 = list_mask
+        s1, s2 = self.attentionLayer(list_attn2,list_attn2,list_attn2,mask)
+
+        return s1
+
+
+
+class TransBase(nn.Module):
+    def __init__(self, args, obs_shape, cat_self=True, attn_internal=False):
+        super(TransBase, self).__init__()
+
+        self._use_feature_normalization = args.use_feature_normalization
+        self._use_orthogonal = args.use_orthogonal
+        self._use_ReLU = args.use_ReLU
+        self._stacked_frames = args.stacked_frames
+        self._layer_N = args.layer_N
+        self.hidden_size = args.hidden_size
+
+        obs_dim = obs_shape[0]
+
+        if self._use_featrue_normalization:
+            self.feature_norm = nn.LayerNorm(obs_dim)
+
+        self.trans = TransLayer(obs_dim, self.hidden_size,
+                              self._layer_N, self._use_orthogonal, self._use_ReLU)
+
+    def forward(self, x):
+        if self._use_feature_normalization:
+            x = self.feature_norm(x)
+
+        x = self.trans(x)
+
+        return x
+
+
+class PositionalEncoding(nn.Module):
+
+    def __init__(self, d_model, dropout=0.1, max_len=5000):
+        super(PositionalEncoding, self).__init__()
+        self.dropout = nn.Dropout(p=dropout)
+
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0).transpose(0, 1)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x):
+        x = x + self.pe[:x.size(0), :]
+        return self.dropout(x)
+
+d_model = Feature
+n_heads= 6
+d_k = d_v = 64
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_model , n_heads, d_k , d_v ):
+        super(MultiHeadAttention, self).__init__()
+        self.W_Q = nn.Linear(d_model, d_k * n_heads, bias=False)
+        self.W_K = nn.Linear(d_model, d_k * n_heads, bias=False)
+        self.W_V = nn.Linear(d_model, d_v * n_heads, bias=False)
+        self.fc = nn.Linear(n_heads * d_v, d_model, bias=False)
+    def forward(self, input_Q, input_K, input_V, attn_mask):
+        '''
+        input_Q: [batch_size, len_q, d_model]
+        input_K: [batch_size, len_k, d_model]
+        input_V: [batch_size, len_v(=len_k), d_model]
+        attn_mask: [batch_size, seq_len, seq_len]
+        '''
+        residual, batch_size = input_Q, input_Q.size(0)
+        # (B, S, D) -proj-> (B, S, D_new) -split-> (B, S, H, W) -trans-> (B, H, S, W)
+        Q = self.W_Q(input_Q).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # Q: [batch_size, n_heads, len_q, d_k]
+        K = self.W_K(input_K).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # K: [batch_size, n_heads, len_k, d_k]
+        V = self.W_V(input_V).view(batch_size, -1, n_heads, d_v).transpose(1,2)  # V: [batch_size, n_heads, len_v(=len_k), d_v]
+
+        attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size, n_heads, seq_len, seq_len]
+
+        # context: [batch_size, n_heads, len_q, d_v], attn: [batch_size, n_heads, len_q, len_k]
+        context, attn = ScaledDotProductAttention()(Q, K, V, attn_mask)
+        context = context.transpose(1, 2).reshape(batch_size, -1, n_heads * d_v) # context: [batch_size, len_q, n_heads * d_v]
+        output = self.fc(context) # [batch_size, len_q, d_model]
+
+        return output + residual, attn
+
+
+class ScaledDotProductAttention(nn.Module):
+    def __init__(self):
+        super(ScaledDotProductAttention, self).__init__()
+
+    def forward(self, Q, K, V, attn_mask):
+        '''
+        Q: [batch_size, n_heads, len_q, d_k]
+        K: [batch_size, n_heads, len_k, d_k]
+        V: [batch_size, n_heads, len_v(=len_k), d_v]
+        attn_mask: [batch_size, n_heads, seq_len, seq_len]
+        '''
+        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k)  # scores : [batch_size, n_heads, len_q, len_k]
+        scores.masked_fill_(attn_mask, -1e9)  # Fills elements of self tensor with value where mask is True.
+
+        attn = nn.Softmax(dim=-1)(scores)
+        context = torch.matmul(attn, V)  # [batch_size, n_heads, len_q, d_v]
+        return context, attn
+
+
+def get_attn_subsequence_mask(seq):
+    '''
+    seq: [batch_size, tgt_len]
+    '''
+    attn_shape = [seq.size(0), seq.size(1), seq.size(1)]
+    subsequence_mask = np.triu(np.ones(attn_shape), k=1) # Upper triangular matrix
+    subsequence_mask = torch.from_numpy(subsequence_mask).byte()
+    return subsequence_mask # [batch_size, tgt_len, tgt_len]
+
+
+def get_attn_pad_mask(seq_q, seq_k):
+    '''
+    seq_q: [batch_size, seq_len]
+    seq_k: [batch_size, seq_len]
+    seq_len could be src_len or it could be tgt_len
+    seq_len in seq_q and seq_len in seq_k maybe not equal
+    '''
+    batch_size, len_q,_ = seq_q.size()
+
+    batch_size, len_k,_ = seq_k.size()
+    # eq(zero) is PAD token
+    data = []
+    for i in range (0,len_q):
+        data.append(False)
+    data2 = []
+    for i in range (0,len_k):
+        data2.append(data)
+    data3 = []
+    data3.append(data2)
+
+    return torch.tensor(data3)
+
+
+class new_mlp_gpt2(nn.Module):
+    def __init__(self, input_dim, embd_dim, output_dim, config):
+        super(new_mlp_gpt2, self).__init__()
+
+        self.linear1 = nn.Linear(input_dim, embd_dim)
+
+        self.gpt2 = GPT2Model(config)
+
+        self.linear2 = nn.Linear(embd_dim, 128)
+
+    def forward(self, x):
+
+
+        x = self.linear1(x)
+        gpt2_output = self.gpt2(inputs_embeds=x)
+
+        x = gpt2_output.last_hidden_state
+        x = self.linear2(x)
+
+
+        return x

config.py

@@ -166,11 +166,11 @@ def get_config():
         default="check",
         help="an identifier to distinguish different experiment.",
     )
-    parser.add_argument("--seed", type=int, default=1, help="Random seed for numpy/torch")
+    parser.add_argument("--seed", type=int, default=10, help="Random seed for numpy/torch")
     parser.add_argument(
         "--cuda",
         action="store_false",
-        default=True,
+        default=False,
         help="by default True, will use GPU to train; or else will use CPU;",
     )
     parser.add_argument(
@@ -182,13 +182,13 @@ def get_config():
     parser.add_argument(
         "--n_training_threads",
         type=int,
-        default=2,
+        default=10,
         help="Number of torch threads for training",
     )
     parser.add_argument(
         "--n_rollout_threads",
         type=int,
-        default=5,
+        default=1,
         help="Number of parallel envs for training rollouts",
     )
     parser.add_argument(
@@ -206,7 +206,7 @@ def get_config():
     parser.add_argument(
         "--num_env_steps",
         type=int,
-        default=10e6,
+        default=2*10e6,
         help="Number of environment steps to train (default: 10e6)",
     )
     parser.add_argument(
@@ -256,13 +256,13 @@ def get_config():
     parser.add_argument(
         "--hidden_size",
         type=int,
-        default=64,
+        default=128,
         help="Dimension of hidden layers for actor/critic networks",
     )
     parser.add_argument(
         "--layer_N",
         type=int,
-        default=1,
+        default=3,
         help="Number of layers for actor/critic networks",
     )
     parser.add_argument("--use_ReLU", action="store_false", default=True, help="Whether to use ReLU")
@@ -314,11 +314,11 @@ def get_config():
     )
 
     # optimizer parameters
-    parser.add_argument("--lr", type=float, default=5e-4, help="learning rate (default: 5e-4)")
+    parser.add_argument("--lr", type=float, default=1e-5, help="learning rate (default: 5e-4)")
     parser.add_argument(
         "--critic_lr",
         type=float,
-        default=5e-4,
+        default=1e-4,
         help="critic learning rate (default: 5e-4)",
     )
     parser.add_argument(

envs/AerialVehicle.py

+import math
+import random
+
+import numpy as np
+
+
+class AerialVehicle(object):
+    def __init__(self, CustomerNum, TotalContentNum, ContentSize, Hight,
+                 EnvA, EnvB, Frequency, Bandwidth, TransmitPower, SpeedOfLight,
+                 AvgLOS, AvgNLOS, ConstrainLOS, Noise, MaxPlaceX, MinPlaceX,
+                 MaxPlaceY, MinPlaceY):
+        self.CustomerNum = CustomerNum          # 区域内用户数量
+        self.TotalContentNum = TotalContentNum  # 文件总数量
+        self.ContentSize = ContentSize          # 单个文件大小
+        self.K = 0                              # 分片数量
+        self.PlaceX = 0                         # 无人机位置水平坐标x
+        self.PlaceY = 0                         # 无人机位置水平坐标y
+        self.PlaceH = Hight                     # 无人机飞行高度H
+        self.EnvA = EnvA                        # 系统环境参数a
+        self.EnvB = EnvB                        # 系统环境参数b
+
+        self.Frequency = Frequency              # 无人机载波频率
+        self.SpeedOfLight = SpeedOfLight        # 光速取值
+        self.AvgLOS = AvgLOS                    # 无人机LOS传输平均额外损失
+        self.AvgNLOS = AvgNLOS                  # 无人机NLOS传输平均额外损失
+        self.ConstrainLOS = ConstrainLOS        # 最低LOS概率限制
+
+        self.Bandwidth = Bandwidth              # 无人机总带宽
+        self.TransmitPower = TransmitPower      # 无人机传输总功率
+        self.Noise = Noise                      # 高斯白噪声
+
+        self.ServiceNum = 0                     # 本轮服务的数量
+        self.ServiceList = []                   # 本轮服务的用户List
+
+        self.MaxPlaceX = MaxPlaceX              # 区域范围
+        self.MinPlaceX = MinPlaceX
+        self.MaxPlaceY = MaxPlaceY
+        self.MinPlaceY = MinPlaceY
+
+        self.TotalContentNum = TotalContentNum
+
+        self.ServiceRadius = self.getRadiusOfUAV()                         # 无人机服务半径
+        print("RadiusOfUAV " +  str(self.ServiceRadius))
+        self.CachedContentList = [0 for _ in range(TotalContentNum)]       # 无人机缓存列表F
+
+    def cacheContent(self, i):
+        self.CachedContentList[i] = 1
+
+    def clearCacheContent(self):
+        self.CachedContentList = [0 for _ in range(self.TotalContentNum)]
+
+    def whetherCacheContent(self, i):
+        return self.CachedContentList[i] == 1
+
+    def getCacheList(self):
+        return self.CachedContentList
+
+    def setK(self, K):
+        self.K = K
+
+    def getPlaceX(self):
+        return self.PlaceX
+
+    def getPlaceY(self):
+        return self.PlaceY
+
+    # 移动无人机位置
+    def moveTo(self,x, y):
+        self.PlaceX = x
+        self.PlaceY = y
+
+    def moveToByDist(self, dist, direction):
+        x = self.PlaceX + dist * math.cos(direction)
+        y = self.PlaceY + dist * math.sin(direction)
+
+        if self.isBeyond(x, y):
+            return 5
+        self.PlaceX = x
+        self.PlaceY = y
+        return self.InPlace()
+
+    def isBeyond(self, x, y):
+        if x > self.MaxPlaceX or x < self.MinPlaceX or y > self.MaxPlaceY or y < self.MinPlaceY:
+            return True
+        return False
+
+    # 限制无人机飞行范围
+    def InPlace(self):
+        punish = 0
+        if self.PlaceX > self.MaxPlaceX:
+        #    self.PlaceX = self.MaxPlaceX
+            punish+=3
+        if self.PlaceX > self.MaxPlaceX - 5:
+            punish += 1
+        if self.PlaceX < self.MinPlaceX:
+            #   self.PlaceX = self.MinPlaceX
+            punish += 3
+        if self.PlaceX < self.MinPlaceX + 5:
+            punish += 1
+        if self.PlaceY > self.MaxPlaceY:
+            #    self.PlaceY = self.MaxPlaceY
+            punish += 3
+        if self.PlaceY > self.MaxPlaceY - 5:
+            punish += 1
+        if self.PlaceY < self.MinPlaceY:
+            #    self.PlaceY = self.MinPlaceY
+            punish += 3
+        if self.PlaceY < self.MinPlaceY + 5:
+            punish += 1
+        return punish
+
+    # 计算水平距离
+    def getDist(self,x, y):
+        return math.sqrt(math.pow(self.PlaceX - x, 2) + math.pow(self.PlaceY - y, 2))
+
+    # 计算LOS路径的概率
+    def getPossOfLos(self, x, y):
+        var1 = 180 / math.pi * math.atan(self.PlaceH / self.getDist(x, y))
+        var2 = - self.EnvB * (var1 - self.EnvA)
+        var3 = 1 + self.EnvA * math.exp(var2)
+        return 1 / var3
+
+    # 计算LOS路径的概率
+    def getPossOfLosByDist(self, dist):
+        var1 = (180)/3.1415 * math.atan(self.PlaceH / dist)
+        var2 = - self.EnvB * (var1 - self.EnvA)
+        var3 = 1 + self.EnvA * math.exp(var2)
+        return 1 / var3
+
+    # 计算路损的公共部分
+    def getNormalLoss(self, x, y):
+        var1 = math.log10(4 * math.pi * self.Frequency * self.getDist(x, y) / self.SpeedOfLight)
+        if var1 <= 0:
+            return 0
+        return 20 * var1
+
+    # 计算LOS路损
+    def getLOS(self, x, y):
+        return self.getNormalLoss(x, y) + self.AvgLOS
+
+    # 计算NLOS路损
+    def getNLOS(self, x, y):
+        return self.getNormalLoss(x, y) + self.AvgNLOS
+
+    # 计算平均路损
+    def getAvgLOS(self, x, y):
+        return self.getLOS(x, y) * self.getPossOfLos(x, y) + self.getNLOS(x, y) * (1 - self.getPossOfLos(x, y))
+
+    # 计算无人机服务半径
+    def getRadiusOfUAV(self):
+        var1 = math.log10((1 - self.ConstrainLOS) / (self.EnvA * self.ConstrainLOS))
+        var2 = math.tan(self.EnvA - (1 / self.EnvB) * var1)
+        return self.PlaceH / var2
+
+    # 无人机分配传输功率
+    def allocTransmitPower(self):
+        return self.TransmitPower
+
+    # 无人机分配带宽
+    def allocBandWidth(self):
+        return self.Bandwidth / self.ServiceNum
+
+    # 尝试是否在无人机服务范围内
+    def tryGetService(self, x, y, requestIndex, UserIndex):
+        print("tryGetService")
+        if self.getDist(x, y) > self.ServiceRadius:
+           # 水平距离大于服务半径
+           return False
+        elif not self.whetherCacheContent(requestIndex):
+            # 该无人机未缓存该内容
+           print("nonononono")
+           return False
+        else:
+            self.ServiceNum = self.ServiceNum + 1
+            self.ServiceList.append(UserIndex)
+            return True
+
+    def addService(self, UserIndex):
+        self.ServiceNum = self.ServiceNum + 1
+        self.ServiceList.append(UserIndex)
+
+
+    # 计算下行传输速率
+    def getTransSpeed(self, x, y):
+        var1 = self.allocTransmitPower() / (self.Noise * np.power(10, self.getAvgLOS(x, y) / 10))
+        return self.allocBandWidth() * math.log2(var1 + 1) / (1024 * 1024 * 8)
+
+    # 计算分片大小
+    def getSizeOFSlice(self):
+        return self.ContentSize / self.K
+
+    # 能否从无人机处获得分片
+    def tryGetContent(self, x, y, time, UserIndex, transSpeedBaseLine, requestIndex):
+        if self.CachedContentList[requestIndex] != 1:
+            return -1
+        if UserIndex not in self.ServiceList:
+            return -1
+        speed = self.getTransSpeed(x, y)
+
+         
+        if speed < transSpeedBaseLine:
+            return False
+        if (self.getSizeOFSlice() / speed) > time:
+            return False
+        return True
+
+    def clearServiceList(self):
+        self.ServiceNum = 0
+        return self.ServiceList.clear()
+
+
+
+
+
+####################################leader
+    def setLeader(self, Role):
+        self.Role = Role
+
+    def isLeader(self):
+        return self.Role
+
+
+
+
+
+
+

envs/UserForUAV.py

+import random
+from numpy import random as r
+
+class UserForUAV(object):
+
+    def __init__(self, UserId, TotalContentNum, FalvorNum, K):
+        self.UserId = UserId                     # 用户索引
+        self.TotalContentNum = TotalContentNum   # 文件集合总数
+        self.FalvorNum = FalvorNum               # 该用户倾向选择的文件数量
+
+        self.PlaceX = 0                          # 用户水平位置x
+        self.PlaceY = 0                          # 用户水平位置y
+
+        self.K = K
+        self.RequestIndex = 0
+
+        self.UAVList = []
+
+        if FalvorNum > TotalContentNum:
+            self.FalvorNum = TotalContentNum
+
+        self.FalvorList = random.sample(range(1, 9), FalvorNum)    # 该用户倾向选择的文件
+
+    # 按照zipf分布，模拟用户请求
+    def genRequestIndex(self):
+        x = r.zipf(a=2, size=1)[0]
+        x = x - 1
+        if x >= len(self.FalvorList):
+            self.RequestIndex = self.FalvorList[len(self.FalvorList)-1]
+        else:
+            self.RequestIndex = self.FalvorList[x]
+        return self.RequestIndex
+
+    def getRequestIndex(self):
+        return self.RequestIndex
+
+    # 移动用户位置
+    def moveTo(self,x, y):
+        self.PlaceX = x
+        self.PlaceY = y
+
+    def setUAVList(self, UAVList):
+        self.UAVList = UAVList
+
+    def tryGetservice(self):
+
+        for i in range(0, len(self.UAVList)):
+            self.UAVList[i].tryGetService(self.PlaceX, self.PlaceY, self.RequestIndex, self.UserId)
+
+    def tryGetCache(self, transSpeedBaseLine, time):
+        vehiclesAbleToTrans = 0
+        self.genRequestIndex()
+        for i in range(0, len(self.UAVList)):
+            if self.UAVList[i].tryGetContent(self.PlaceX, self.PlaceY, time, self.UserId, transSpeedBaseLine,  self.RequestIndex):
+                vehiclesAbleToTrans = vehiclesAbleToTrans + 1
+        return vehiclesAbleToTrans, self.RequestIndex
+
+
+
+
+

envs/env_continuous.py

@@ -2,6 +2,7 @@ import gym
 from gym import spaces
 import numpy as np
 from envs.env_core import EnvCore
+from envs.env_discrete import MultiDiscrete
 
 
 class ContinuousActionEnv(object):
@@ -12,10 +13,11 @@ class ContinuousActionEnv(object):
 
     def __init__(self):
         self.env = EnvCore()
-        self.num_agent = self.env.agent_num
+        self.num_agent = self.env.AerialVehiclesNum + 2
 
         self.signal_obs_dim = self.env.obs_dim
         self.signal_action_dim = self.env.action_dim
+        print( self.num_agent)
 
         # if true, action is a number 0...N, otherwise action is a one-hot N-dimensional vector
         self.discrete_action_input = False
@@ -29,7 +31,8 @@ class ContinuousActionEnv(object):
 
         share_obs_dim = 0
         total_action_space = []
-        for agent in range(self.num_agent):
+
+        for agent in range(self.env.AerialVehiclesNum):
             # physical action space
             u_action_space = spaces.Box(
                 low=-np.inf,
@@ -54,7 +57,60 @@ class ContinuousActionEnv(object):
                     dtype=np.float32,
                 )
             )  # [-inf,inf]
+        print('section1')
+
+        mu_action = []
+        for i in range(self.env.CustomerNum * self.env.AerialVehiclesNum):
+           mu_action.append([0, 1])
+        u_action_space = MultiDiscrete(mu_action)
+
+        if self.movable:
+            total_action_space.append(u_action_space)
+
+        # total action space
+        self.action_space.append(u_action_space)
+
+        # observation space
+        share_obs_dim += self.env.getDimension2()
+        self.observation_space.append(
+            spaces.Box(
+                low=-np.inf,
+                high=+np.inf,
+                shape=(self.env.getDimension2(),),
+                dtype=np.float32,
+            )
+        )
+
+        print('section2')
+
+
+        mu_action = []
+
+        mu_action.append([0, self.env.ContentNum - 1])
+        u_action_space = spaces.Discrete(self.env.ContentNum)
 
+        if self.movable:
+            total_action_space.append(u_action_space)
+
+            # total action space
+        self.action_space.append(u_action_space)
+
+        # observation space
+        share_obs_dim += (self.env.CacheNum + self.env.CustomerNum)
+        self.observation_space.append(
+            spaces.Box(
+                low=-np.inf,
+                high=+np.inf,
+                shape=(self.env.CacheNum + self.env.CustomerNum,),
+                dtype=np.float32,
+            )
+        )
+
+
+
+
+        print("ons")
+        print(share_obs_dim)
         self.share_observation_space = [
             spaces.Box(
                 low=-np.inf, high=+np.inf, shape=(share_obs_dim,), dtype=np.float32
@@ -62,6 +118,7 @@ class ContinuousActionEnv(object):
             for _ in range(self.num_agent)
         ]
 
+
     def step(self, actions):
         """
         输入actions维度假设：
@@ -75,11 +132,11 @@ class ContinuousActionEnv(object):
 
         results = self.env.step(actions)
         obs, rews, dones, infos = results
-        return np.stack(obs), np.stack(rews), np.stack(dones), infos
+        return obs, np.stack(rews), np.stack(dones), infos
 
     def reset(self):
         obs = self.env.reset()
-        return np.stack(obs)
+        return obs
 
     def close(self):
         pass

envs/env_core.py

+import copy
+import math
+
 import numpy as np
+from matplotlib import pyplot as plt
+
+from envs.AerialVehicle import AerialVehicle
+from numpy import random
+import torch
+from envs.UserForUAV import UserForUAV
+
+CustomerNum = 30
+AerialVehiclesNum = 7
+TotalContentNum = 30
+MaxPlaceX = 500
+MaxPlaceY = 500
+Move = MaxPlaceX / 100
+ContentSize = 256
+CacheLimit = 3600
+Hight = 200
+EnvA = 11.9
+EnvB = 0.13
+Frequency = 2e9
+Bandwidth = 40e6
+TransmitPower = 2
+SpeedOfLight = 3e8
+AvgLOS = 6
+AvgNLOS = 20
+ConstrainLOS = 0.02
+Noise = 1e-13
+DownSize = 0
+
+FalvorNum = 2
+K = 4
 
+transSpeedBaseLine = 0.5
+timeLImit = 20
 
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+HID=[256,256,256]
+HID_SIZE = 3
+LOG_SIG_MAX = 2#max_log
+LOG_SIG_MIN = -20#min_log
+alpha=0.036
+MIN_K = 2
+RANDOM_STEP = 2000
 class EnvCore(object):
-    """
-    # 环境中的智能体
-    """
+    dimension1 = AerialVehiclesNum + 1
+    dimension2 = 2 * AerialVehiclesNum
+    dimension3 = int(CacheLimit // (ContentSize / K)) + CustomerNum
+    dimension4 = 2* AerialVehiclesNum +  CustomerNum + TotalContentNum
+    AerialVehiclesNum = AerialVehiclesNum
+
 
     def __init__(self):
-        self.agent_num = 2  # 设置智能体(小飞机)的个数，这里设置为两个 # set the number of agents(aircrafts), here set to two
-        self.obs_dim = 14  # 设置智能体的观测维度 # set the observation dimension of agents
-        self.action_dim = 5  # 设置智能体的动作维度，这里假定为一个五个维度的 # set the action dimension of agents, here set to a five-dimensional
 
-    def reset(self):
-        """
-        # self.agent_num设定为2个智能体时，返回值为一个list，每个list里面为一个shape = (self.obs_dim, )的观测数据
-        # When self.agent_num is set to 2 agents, the return value is a list, each list contains a shape = (self.obs_dim, ) observation data
-        """
-        sub_agent_obs = []
-        for i in range(self.agent_num):
-            sub_obs = np.random.random(size=(14,))
-            sub_agent_obs.append(sub_obs)
-        return sub_agent_obs
+        self.CustomerNum = CustomerNum
+        self.AerialVehiclesNum = AerialVehiclesNum
+        # StartSystem+
+        self.agent_num = AerialVehiclesNum
+
+        self.MaxPlaceX = MaxPlaceX
+        self.MaxPlaceY = MaxPlaceY
+        self.PlaceList = []
+
+        self.MinPlaceX = 0
+        self.MinPlaceY = 0
+
+        self.k = K
+        self.Move = Move
+
+        self.FirstStep = True
+
+        xDelta = self.MaxPlaceX - self.MinPlaceX
+        yDelta = self.MaxPlaceY - self.MinPlaceY
+        self.initX = xDelta * np.random.uniform(0, 1,
+                                                self.AerialVehiclesNum) + self.MinPlaceX  # x coordinates of Poisson points
+        self.initY = yDelta * np.random.uniform(0, 1, self.AerialVehiclesNum) + self.MinPlaceY
+
+        self.state = []
+        self.action = []
+        self.obs_dim = self.getStateDimension()  # 设置智能体的观测维度 # set the observation dimension of agents
+        self.action_dim = self.getActionDimension()
+
+        self.AerialVehicles = [self.newUAV() for _ in range(AerialVehiclesNum)]
+        self.Customers = [self.newUser(i, self.AerialVehicles) for i in range(CustomerNum)]
+
+        self.resetCustomer()
+        self.resetUAV()
+
+        self.nowStep = 0
+        self.epi = 0
+
+        self.bags = 0
+        self.slices = 0
+
+        self.allBags = []
+        self.allSlices = []
+        self.ks = []
+        self.epis = []
+
+        self.ContentNum = TotalContentNum
+        self.CacheLimit = CacheLimit
+        size = ContentSize / self.k
+        self.CacheNum = int(self.CacheLimit // size)
+
+        self.CacheContent = [0 for _ in range(  self.CacheNum )]
+        self.CacheNumNow = 0
+        self.CacheDict = {}
+        self.Recommend = 0
+
+        self.AllocReward = 0
+        self.DownSize = DownSize
+
+        self.dimension1 = self.getDimension1()
+        self.dimension2 = self.getDimension2()
+        self.dimension3 = self.getDimension3()
+        self.dimension4 = self.getDimension4()
+        print("11111111111111111111111")
+        print(self.dimension1)
+        print(self.dimension2)
+        print(self.dimension3)
+        print(self.dimension4)
+        print(self.dimension1*5 +  self.dimension2 +  self.dimension3 +  self.dimension4)
+
+
+
+
+
+
+
+
+    def resetUAV(self):
+        self.resetUAVPosition()
+        self.resetUAVNumK()
+        self.resetServiceList()
+
+    def resetUAVCache(self):
+        for i in range(0, len(self.AerialVehicles)):
+            for j in range(0, TotalContentNum):
+                self.AerialVehicles[i].cacheContent(j)
+
+    def resetUAVPosition(self):
+        self.RandomUAVPosition()
+
+    # 重置K值，目前先固定
+    def resetUAVNumK(self):
+        self.setUAVNumK(K)
+
+    def setUAVNumK(self, k):
+        self.k = k
+        for i in range(0, len(self.AerialVehicles)):
+            self.AerialVehicles[i].setK(k)
+
 
     def step(self, actions):
-        """
-        # self.agent_num设定为2个智能体时，actions的输入为一个2纬的list，每个list里面为一个shape = (self.action_dim, )的动作数据
-        # 默认参数情况下，输入为一个list，里面含有两个元素，因为动作维度为5，所里每个元素shape = (5, )
-        # When self.agent_num is set to 2 agents, the input of actions is a 2-dimensional list, each list contains a shape = (self.action_dim, ) action data
-        # The default parameter situation is to input a list with two elements, because the action dimension is 5, so each element shape = (5, )
-        """
         sub_agent_obs = []
         sub_agent_reward = []
         sub_agent_done = []
         sub_agent_info = []
-        for i in range(self.agent_num):
-            sub_agent_obs.append(np.random.random(size=(14,)))
-            sub_agent_reward.append([np.random.rand()])
+        self.resetServiceList()
+
+        punish = []
+
+        for i in range(0, len(self.AerialVehicles)):
+            act = actions[i]
+            dist = act[0] * self.Move
+            pi = act[1] * math.pi
+            punish.append(self.AerialVehicles[i].moveToByDist(dist, pi))
+
+        self.addService(actions[self.AerialVehiclesNum])
+        self.resetCache(actions[self.AerialVehiclesNum + 1])
+
+
+        totalReward = 0
+        num = 0
+        cacheMit = 0
+        for i in range(0, len(self.Customers)):
+            reward, requestIndex = self.Customers[i].tryGetCache(transSpeedBaseLine, timeLImit)
+            reward = reward
+            totalReward += reward
+            if requestIndex in self.CacheContent:
+                cacheMit += 1
+                self.CacheDict[requestIndex] = self.CacheDict[requestIndex] + 1
+            if reward >= self.k:
+                num += 1
+        totalReward = totalReward * 1 / self.k
+        self.nowStep += 1
+
+        self.bags += num
+        self.slices += totalReward
+   #     print(self.k)
+        if self.nowStep % 200 == 0:
+           print(self.bags)
+           print(self.slices)
+           print(self.CacheContent)
+
+           self.allBags.append(self.bags)
+           self.allSlices.append(self.slices)
+           self.ks.append(self.k)
+           self.epis.append(self.epi)
+           self.epi += 1
+
+
+
+           if self.epi % 200 == 0:
+               self.plotPic(self.epis, self.allBags)
+               self.plotPic(self.epis, self.allSlices)
+               self.plotPic(self.epis, self.ks)
+
+               print(self.allBags)
+               print(self.allSlices)
+
+           if self.epi > 500 :
+               self.DownSize = 0
+           if self.epi > 550 :
+               self.DownSize = 0
+
+
+
+
+           self.bags = 0
+           self.slices = 0
+
+
+        totalReward = totalReward + num*50
+
+        for i in range(0, len(self.AerialVehicles)):
+            sub_agent_reward.append([totalReward - punish[i]*10])
+            sub_agent_obs.append(self.getStateForUAVAgent(i))
             sub_agent_done.append(False)
-            sub_agent_info.append({})
 
+        sub_agent_reward.append([totalReward + self.AllocReward])
+        sub_agent_obs.append(self.getStateForMatchAgent())
+        sub_agent_done.append(False)
+
+        sub_agent_reward.append([totalReward+cacheMit*0.1+self.RecommendReward()])
+        sub_agent_obs.append(self.getStateForCacheAgent())
+        sub_agent_done.append(False)
+
+        sub_agent_info.append({})
+
+        #print(self.nowStep)
+        #print(self.resetState2())
+        #print(sub_agent_obs)
         return [sub_agent_obs, sub_agent_reward, sub_agent_done, sub_agent_info]
+
+    def addService(self, act):
+        self.AllocReward = 0
+        reward = [0 for _ in range(TotalContentNum)]
+
+        for i in range(0, len(self.Customers)):
+            for j in range(0, len(self.AerialVehicles)):
+                num = i * len(self.AerialVehicles) + j
+                if act[2 * num] == 1:
+                  #  print('yres')
+                    self.AerialVehicles[j].addService(i)
+                    reward[i] = reward[i] + 1
+
+        for i in range(0, len(reward)):
+            if reward[i] >= self.k:
+                self.AllocReward = self.AllocReward + 1
+
+    def RecommendReward(self):
+        if self.Recommend < 10:
+            return 10
+        return 0
+
+    def resetCache(self, act):
+        index = self.findCache(act)
+        self.Recommend = index
+        if index not in self.CacheContent:
+            self.Recommend = index
+            if self.CacheNumNow < self.CacheNum:
+                self.CacheContent[self.CacheNumNow] = index
+                self.CacheNumNow = self.CacheNumNow + 1
+            else:
+                j = self.findLeast()
+                for i in range(0, len(self.CacheContent)):
+                    if j == self.CacheContent[i]:
+                        self.CacheContent[i] = index
+
+        contents = self.CacheContent
+
+        self.CacheDict.clear()
+        for j in range(0, len(self.AerialVehicles)):
+            self.AerialVehicles[j].clearCacheContent()
+        for i in range(0, len(contents)):
+            self.CacheDict[contents[i]] = 0
+            for j in range(0, len(self.AerialVehicles)):
+                self.AerialVehicles[j].cacheContent(contents[i])
+
+        for i in range(0, self.DownSize):
+            self.AerialVehicles[i].clearCacheContent()
+
+    def findLeast(self):
+        least = 10000
+        res = 0
+        for key in self.CacheDict.keys():
+            if self.CacheDict[key] < least:
+                least = self.CacheDict[key]
+                res = key
+        return res
+    def findCache(self, act):
+        index = 0
+        for j in range(0, len(act)):
+            if act[j] == 1 :
+                return j
+        return index
+
+    def findIndex(self, score, num):
+        t = copy.deepcopy(score)
+        # 求m个最大的数值及其索引
+        max_index = []
+        for _ in range(num):
+            index = np.argmax(t)
+            t[index] = -1
+            max_index.append(index)
+        return max_index
+
+
+    def getStateForMatchAgent(self):
+        state = []
+
+        # 无人机位置部分
+        for i in range(0, len(self.AerialVehicles)):
+            state.append(self.AerialVehicles[i].getPlaceX()/self.MaxPlaceX)
+            state.append(self.AerialVehicles[i].getPlaceY()/self.MaxPlaceX)
+
+            # 用户请求部分
+        for i in range(0, len(self.Customers)):
+            state.append(self.Customers[i].getRequestIndex())
+
+        for i in range(0, TotalContentNum):
+            if self.AerialVehicles[0].whetherCacheContent(i):
+                state.append(1)
+            else:
+                state.append(0)
+
+
+
+        return state
+
+    def getStateForCacheAgent(self):
+        state = []
+        for i in range(0, len(self.CacheContent)):
+            state.append(self.CacheContent[i])
+
+        for i in range(0, len(self.Customers)):
+            state.append(self.Customers[i].getRequestIndex())
+
+        return state
+
+    def resetState2(self):
+        self.state = []
+
+        # 无人机位置部分
+        for i in range(0, len(self.AerialVehicles)):
+            self.state.append(self.AerialVehicles[i].getPlaceX() /self.MaxPlaceX)
+            self.state.append(self.AerialVehicles[i].getPlaceY() /self.MaxPlaceX)
+        # 用户请求部分
+        for i in range(0, len(self.Customers)):
+            self.state.append(self.Customers[i].getRequestIndex())
+        # 缓存LIST
+        for i in range(0, TotalContentNum):
+            if self.AerialVehicles[0].whetherCacheContent(i):
+                self.state.append(1)
+            else:
+                self.state.append(0)
+
+        return self.state
+
+    def getStateDimension2(self):
+        return 2 * len(self.AerialVehicles) + len(self.Customers) + TotalContentNum
+
+    def reset(self):
+        self.RandomUAVPosition()
+        self.resetServiceList()
+
+        sub_agent_obs = []
+
+
+        for i in range(0, len(self.AerialVehicles)):
+            sub_agent_obs.append(self.getStateForUAVAgent(i))
+
+        sub_agent_obs.append(self.getStateForMatchAgent())
+        sub_agent_obs.append(self.getStateForCacheAgent())
+
+
+        return sub_agent_obs
+
+
+
+
+    # 重建state
+    def getStateForUAVAgent(self, index):
+        # 无人机位置部分
+        s = []
+        s.append(self.AerialVehicles[index].getPlaceX()/self.MaxPlaceX)
+        s.append(self.AerialVehicles[index].getPlaceY()/self.MaxPlaceX)
+        for j in range(0, len(self.AerialVehicles)):
+            if index == j:
+                continue
+            s.append(self.AerialVehicles[index].getDist(self.AerialVehicles[j].getPlaceX(), self.AerialVehicles[j].getPlaceY()) /self.MaxPlaceX)
+
+        return s
+
+    def getStateDimension(self):
+        return self.AerialVehiclesNum + 1
+
+    def getDimension1(self):
+        return self.AerialVehiclesNum + 1
+
+    def getDimension2(self):
+        return 2*self.AerialVehiclesNum + self.CustomerNum + TotalContentNum
+
+    def getDimension3(self):
+        return len(self.CacheContent) + self.CustomerNum
+
+    def getDimension4(self):
+        return 2*self.AerialVehiclesNum + self.CustomerNum + TotalContentNum
+
+    def getActionDimension(self):
+        return 2
+
+
+    def resetCustomer(self):
+        self.resetCustomerPosition()
+
+    # 重置无人机状态，后续换为agent，故分开写
+
+    # 随机分配无人机位置
+    def RandomUAVPosition(self):
+        for i in range(0, len(self.AerialVehicles)):
+            self.AerialVehicles[i].moveTo(self.initX[i], self.initY[i])
+
+
+
+    # 清空服务列表
+    def resetServiceList(self):
+        for i in range(0, len(self.AerialVehicles)):
+            self.AerialVehicles[i].clearServiceList()
+
+    def resetCustomerPosition(self):
+        xDelta = self.MaxPlaceX - self.MinPlaceX
+        yDelta = self.MaxPlaceY - self.MinPlaceY
+        xx = [342.67990918, 476.6966731, 1.97413316, 256.09613169, 406.31048083,
+              306.26303341, 360.87765872, 145.93803409, 458.88706126, 357.2878917,
+              271.27218401, 71.0850238, 186.67038003, 337.06680753, 220.91658721,
+              217.00699667, 308.88348923, 256.56912128, 325.19859097, 300.5194767]
+        yy = [402.61159842, 260.8235762, 454.3244404, 159.61804449, 45.22967464,
+              150.35002832, 56.99218093, 414.34066315, 23.44815969, 313.14357416,
+              273.79307796, 409.64349784, 99.47376984, 428.42515123, 175.82631972,
+              377.32384576, 147.98085344, 441.96823978, 162.75581892, 82.50794886]
+        xx = xDelta * np.random.uniform(0, 1, self.CustomerNum) + self.MinPlaceX # x coordinates of Poisson points
+        yy = yDelta * np.random.uniform(0, 1, self.CustomerNum) + self.MinPlaceY
+        print(xx)
+        print(yy)
+
+        for i in range(0, len(self.Customers)):
+            self.Customers[i].moveTo(xx[i], yy[i])
+
+    def newUAV(self):
+        return AerialVehicle(CustomerNum, TotalContentNum, ContentSize, Hight,
+                 EnvA, EnvB, Frequency, Bandwidth, TransmitPower, SpeedOfLight,
+                 AvgLOS, AvgNLOS, ConstrainLOS, Noise, self.MaxPlaceX, self.MinPlaceX,
+                self.MaxPlaceY, self.MinPlaceY)
+
+    def newUser(self, UserId, AerialVehicles):
+        user = UserForUAV(UserId, TotalContentNum, FalvorNum, K)
+        user.setUAVList(AerialVehicles)
+        return user
+
+    def plotPic(self, x, y):
+        plt.xlabel('episode')
+        plt.ylabel('reward')
+        plt.plot(x, y)
+        # You can specify a rotation for the tick labels in degrees or with keywords.
+        #   plt.xticks(x, labels, rotation='vertical')
+        plt.show()
+
+
+
+#r = EnvCore()
+#r.AerialVehicles[0].moveTo(0,0)
+
+#r.AerialVehicles[0].addService(1)
+#r.AerialVehicles[0].addService(2)
+#r.AerialVehicles[0].addService(3)
+#r.AerialVehicles[0].addService(4)
+#s = r.AerialVehicles[0].getTransSpeed(500, 0)
+#print(s)
\ No newline at end of file

envs/env_wrappers.py

@@ -47,7 +47,7 @@ class DummyVecEnv():
 
     def reset(self):
         obs = [env.reset() for env in self.envs] # [env_num, agent_num, obs_dim]
-        return np.array(obs)
+        return np.array(obs, dtype=object)
 
     def close(self):
         for env in self.envs:

envs/plot.py

+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+from matplotlib.ticker import MultipleLocator
+from torch import nn
+
+
+class plot(object):
+    def plotZheXian(self):
+        x_axis_data = [2, 3, 4, 5]  # x
+        y_axis_data = [0.47, 0.67, 0.61, 0.59]  # y
+        y_axis_data2 = [0.42, 0.59, 0.58, 0.42]  # y
+        y_axis_data3 = [0.32, 0.46, 0.45, 0.31]  # y
+
+        plt.plot(x_axis_data, y_axis_data, 'b*--', alpha=1, linewidth=1, label='customer = 20')  # 'bo-'表示蓝色实线，数据点实心原点标注
+        plt.plot(x_axis_data, y_axis_data2, 'rs--', alpha=0.5, linewidth=1, label='customer = 25')
+        plt.plot(x_axis_data, y_axis_data3, 'go--', alpha=0.5, linewidth=1, label='customer = 30')
+        ## plot中参数的含义分别是横轴值，纵轴值，线的形状（'s'方块,'o'实心圆点，'*'五角星   ...，颜色，透明度,线的宽度和标签 ，
+        x_major_locator = MultipleLocator(1)
+        ax = plt.gca()
+        ax.xaxis.set_major_locator(x_major_locator)
+
+        plt.legend()  # 显示上面的label
+        plt.xlabel('K')  # x_label
+        plt.ylabel('transmit')  # y_label
+
+        # plt.ylim(-1,1)#仅设置y轴坐标范围
+        plt.show()
+
+    def plot_col(self):
+        plt.rcParams["figure.dpi"] = 200
+        Random = [0, 0.1, 0.15, 0.2, 0.23]
+        Around = [0, 0.15, 0.25, 0.33, 0.36]
+        Without = [0.01, 0.23, 0.47, 0.53,
+                     0.68]
+        Slice = [0.34, 0.45, 0.62, 0.72,
+                     0.75]
+
+        print(Around)
+        print(Without)
+        print(Random)
+        print(Slice)
+
+        # 计算cost
+        # plt.grid(True, which="major", linestyle="--", color="gray", linewidth=0.75)
+        # plt.grid(True, which="minor", linestyle=":", color="lightgray", linewidth=0.75)
+        labels = ['10', '15', '20', '25', '30']
+        x = np.arange(len(labels))
+        ax = plt.gca()
+
+        ax.set_xticks(x)
+        ax.set_xticklabels(labels)
+
+        width = 0.2
+        plt.xlabel('Day', fontsize=17)
+        plt.ylabel('Cost ($)', fontsize=17)
+        plt.tick_params(labelsize=8)
+        ax.bar(x - 1.5 * width, Random, width, label='Random', color='slategrey', alpha=0.8)
+        ax.bar(x - 0.5 * width, Around, width, alpha=1, label='Around', color='brown')
+        ax.bar(x + 0.5 * width, Without, width, alpha=1, label='Without Slice', color='olive')
+        ax.bar(x + 1.5 * width, Slice, width, label='Slice', color='darkgoldenrod', alpha=0.7)
+
+        # plt.plot(range(len(trainAcc)), trainAcc, label='Average Training Loss', linewidth=1, linestyle='--', marker='.')
+        plt.legend(loc='upper left')
+        plt.legend()
+        plt.savefig(f'D:\p/成本对比实验.jpg')
+        plt.show()
+
+    def plotDemo(self):
+        x_axis_data = [ ]  # x
+        y1 = []
+        y2= []
+        y_axis_data = [1040, 1328, 1255, 1270, 1409, 1320, 1300, 1304, 1323, 1258, 1287, 1260, 1320, 1333, 1259, 1324, 1364, 1328, 1261, 1331, 1409, 1305, 1366, 1396, 1333, 1341, 1363, 1379, 1324, 1396, 1426, 1373, 1329, 1294, 1354, 1335, 1380, 1385, 1401, 1371, 1390, 1386, 1379, 1409, 1470, 1414, 1444, 1459, 1411, 1444, 1420, 1438, 1453, 1417, 1458, 1441, 1442, 1423, 1499, 1438, 1530, 1507, 1482, 1507, 1494, 1540, 1489, 1544, 1516, 1498, 1497, 1512, 1524, 1578, 1582, 1503, 1542, 1535, 1520, 1576, 1541, 1646, 1530, 1582, 1572, 1555, 1570, 1624, 1670, 1609, 1553, 1670, 1624, 1665, 1570, 1558, 1623, 1597, 1680, 1653, 1607, 1674, 1709, 1552, 1657, 1607, 1724, 1669, 1634, 1683, 1663, 1708, 1698, 1658, 1783, 1714, 1707, 1748, 1759, 1762, 1746, 1760, 1730, 1786, 1730, 1724, 1755, 1794, 1784, 1803, 1756, 1697, 1807, 1866, 1876, 1861, 1817, 1820, 1812, 1868, 1779, 1870, 1836, 1872, 1837, 1810, 1873, 1804, 1845, 1843, 1803, 1874, 1882, 1867, 1861, 1858, 1928, 1932, 1942, 1891, 1958, 1945, 1859, 1938, 1907, 1905, 1935, 1929, 1986, 1936, 1968, 1921, 1962, 1939, 2061, 1963, 2002, 1991, 1982, 2041, 1954, 2040, 2003, 1997, 2007, 2036, 2002, 2054, 2075, 2056, 2033, 2092, 1992, 2117, 2076, 2091, 2139, 2118, 2106, 2069, 2121, 2136, 2077, 2128, 2111, 2115, 2055, 2144, 2146, 2105, 2151, 2135, 2205, 2145, 2141, 2129, 2202, 2160, 2198, 2194, 2219, 2171, 2137, 2183, 2180, 2212, 2157, 2252, 2191, 2303, 2197, 2225, 2242, 2260, 2292, 2246, 2218, 2300, 2214, 2239, 2268, 2320, 2240, 2306, 2273, 2280, 2241, 2235, 2246, 2319, 2316, 2300, 2343, 2311, 2282, 2258, 2284, 2297, 2316, 2333, 2362, 2322, 2342, 2369, 2329, 2329, 2332, 2296, 2308, 2295, 2307, 2366, 2353, 2336, 2342, 2355, 2299, 2311, 2341, 2333, 2372, 2322, 2319, 2344, 2319, 2302, 2337, 2306, 2355, 2360, 2380, 2358, 2381, 2364, 2385, 2370, 2337, 2354, 2317, 2394, 2377, 2379, 2348, 2373, 2427, 2412, 2374, 2389, 2361, 2385, 2372, 2443, 2387, 2412, 2373, 2354, 2381, 2378, 2425, 2418, 2393, 2440, 2415, 2447, 2409, 2428, 2457, 2416, 2443, 2441, 2435, 2434, 2460, 2465, 2421, 2406, 2410, 2456, 2458, 2430, 2438, 2456, 2444, 2459, 2488, 2465, 2436, 2446, 2454, 2453, 2445, 2428, 2431, 2452, 2447, 2472, 2459, 2471, 2453, 2455, 2476, 2498, 2466, 2487, 2492, 2476, 2456, 2481, 2476, 2435, 2489, 2459, 2493, 2485, 2504, 2458, 2510, 2486, 2486, 2495, 2484, 2496, 2492, 2494, 2487, 2504, 2483, 2493, 2511, 2489, 2500, 2514, 2493, 2478, 2511, 2490, 2515, 2506, 2528, 2528, 2515, 2507, 2507, 2492, 2520, 2480, 2489, 2526, 2500, 2534, 2531, 2519, 2536, 2536, 2466, 2483, 2523, 2538, 2524, 2522, 2497, 2522, 2514, 2501, 2511, 2519, 2505, 2529, 2526, 2498, 2490, 2522, 2498, 2519, 2510, 2519, 2518, 2512, 2523, 2508, 2496, 2517, 2552, 2536, 2530, 2500, 2544, 2540, 2542, 2549, 2514, 2545, 2522, 2534, 2490, 2547, 2511, 2543, 2536, 2508, 2540, 2538, 2525, 2551, 2528, 2544, 2491, 2530, 2547, 2502, 2519, 2535, 2552, 2546, 2542, 2529, 2523, 2531, 2527, 2543, 2523, 2532, 2530, 2541, 2529, 2520, 2537, 2551, 2528, 2529, 2554, 2495, 2549, 2546, 2560, 2534, 2548, 2538, 2558, 2561, 2546, 2556, 2554, 2531, 2534, 2558, 2559, 2517, 2567, 2512, 2572, 2519, 2550, 2530, 2549, 2555, 2527, 2550, 2536, 2517, 2536, 2547, 2525, 2503, 2486, 2536, 2555, 2552, 2511, 2537, 2546, 2520, 2531, 2523, 2545, 2524, 2527, 2481, 2561, 2501, 2507, 2486, 2491, 2469, 2546, 2561, 2539, 2533, 2525, 2523, 2526, 2523, 2529, 2479, 2516, 2536, 2526, 2558, 2518, 2549, 2555, 2545, 2540, 2565, 2538, 2568, 2526, 2560, 2533, 2561, 2556, 2561, 2566, 2518, 2553, 2575, 2555, 2564, 2555, 2560, 2534, 2565, 2554, 2568, 2539, 2562, 2566, 2540, 2563, 2584, 2561, 2563, 2566, 2566, 2560, 2572, 2563, 2562, 2570, 2562, 2574, 2562, 2563, 2563, 2575, 2562, 2559, 2576, 2555, 2571, 2560, 2541, 2577, 2568, 2561, 2562, 2578, 2577, 2565, 2568, 2571, 2570, 2575, 2561, 2577, 2562, 2569, 2579, 2575, 2570, 2585, 2574, 2579, 2574, 2579, 2583, 2584, 2590, 2571, 2572, 2577, 2575, 2577, 2556, 2587, 2587, 2572, 2570, 2579, 2583, 2589, 2573, 2581, 2574, 2585, 2583, 2590, 2583, 2591, 2593, 2583, 2583, 2577, 2577, 2587, 2587, 2586, 2586, 2572, 2588, 2578, 2545, 2580, 2593, 2576, 2591, 2575, 2578, 2587, 2587, 2589, 2575, 2594, 2578, 2588, 2586, 2582, 2577, 2588, 2575, 2576, 2582, 2584, 2568, 2588, 2587, 2579, 2587, 2600, 2586, 2591, 2576, 2582, 2587, 2578, 2579, 2596, 2578, 2588, 2585, 2579, 2551, 2599, 2600, 2574, 2586, 2580, 2585, 2584, 2573, 2580, 2591, 2596, 2583, 2584, 2595, 2582, 2586, 2586, 2587, 2590, 2585, 2586, 2583, 2586, 2601, 2593, 2587, 2586, 2599, 2595, 2591, 2597, 2590, 2590, 2608, 2590, 2592, 2599, 2589, 2593, 2592, 2584, 2606, 2584, 2581, 2597, 2576, 2587, 2588, 2597, 2587, 2583, 2591, 2600, 2593, 2602, 2598, 2595, 2589, 2599, 2593, 2597, 2599, 2572, 2589, 2587, 2595, 2569, 2592, 2595, 2600, 2608, 2602, 2599, 2588, 2599, 2607, 2608, 2599, 2598, 2597, 2599, 2604, 2592, 2585, 2611, 2598, 2609, 2595, 2606, 2603, 2596, 2594, 2594, 2609, 2599, 2601, 2613, 2613, 2598, 2608, 2596, 2616, 2594, 2616, 2621, 2587, 2607, 2606, 2602, 2622, 2618, 2615, 2608, 2593, 2602, 2598, 2597, 2608, 2602, 2593, 2603, 2608, 2598, 2616, 2616, 2617, 2630, 2630, 2646, 2636, 2613, 2651, 2610, 2609, 2614, 2621, 2618, 2654, 2620, 2604, 2635, 2628, 2598, 2609, 2668, 2615, 2635, 2640, 2640, 2651, 2611, 2660, 2596, 2639, 2636, 2634, 2646, 2667, 2640, 2641, 2636, 2664, 2645, 2640, 2644, 2663, 2643, 2654, 2658, 2661, 2670, 2632, 2672, 2673, 2692, 2675, 2661, 2666, 2693, 2677, 2678, 2693, 2672, 2673, 2692, 2666, 2683, 2684, 2695, 2684, 2686, 2698, 2695, 2692, 2708, 2680, 2678, 2685, 2687, 2704, 2725, 2691, 2695, 2718, 2701, 2732, 2719, 2704, 2721, 2708, 2720, 2714, 2715, 2731, 2734, 2711, 2715, 2716, 2731, 2705, 2736, 2718, 2734, 2736, 2724, 2748, 2739, 2748, 2729, 2735, 2737, 2730, 2758, 2744, 2723, 2740, 2742, 2749, 2742, 2723, 2714, 2741, 2743, 2741, 2760, 2743, 2754, 2737, 2739, 2755, 2751, 2742, 2759, 2753, 2737, 2754, 2760, 2758, 2733, 2751, 2762, 2753, 2763, 2757, 2768, 2766, 2760, 2753, 2765, 2753, 2759, 2764, 2761, 2760, 2772, 2768, 2751, 2764, 2754, 2767, 2778, 2761, 2770, 2781, 2769, 2763, 2769, 2753, 2775, 2775, 2770, 2773, 2773, 2772, 2776, 2773, 2780, 2768, 2770, 2775, 2762, 2781, 2761, 2772, 2770, 2770, 2780, 2776, 2773, 2762, 2768, 2775, 2784, 2775, 2772, 2773, 2776, 2779, 2779, 2780, 2776, 2780, 2779, 2779, 2779, 2778, 2780, 2780, 2774, 2776, 2784, 2777, 2781, 2780, 2779, 2785, 2780, 2779, 2781, 2784, 2782, 2779, 2783, 2782, 2783, 2780, 2780, 2782, 2794, 2781, 2770, 2787, 2786, 2785, 2781, 2791, 2788, 2779, 2746, 2757, 2787, 2779, 2779, 2783, 2779, 2791, 2777, 2790, 2789, 2779, 2794, 2777, 2792, 2789, 2790, 2781, 2786, 2795, 2783, 2790, 2784, 2783, 2791, 2789, 2768, 2785, 2776, 2792, 2791, 2790, 2786, 2786, 2789, 2793, 2791, 2787, 2787, 2786, 2789, 2785, 2792, 2790, 2795, 2787, 2796, 2791, 2790, 2790, 2785, 2781, 2785, 2775, 2790, 2779, 2786, 2793, 2792, 2776, 2786, 2792, 2784, 2795, 2790, 2787, 2787, 2791, 2793, 2794, 2788, 2795, 2784, 2796, 2798, 2786, 2791, 2798, 2792, 2791, 2790, 2791, 2792, 2789, 2791, 2797, 2790, 2795, 2790, 2787, 2784, 2793, 2779, 2789, 2790, 2795, 2791, 2794, 2790, 2793, 2795, 2791, 2794, 2792, 2796, 2794, 2792, 2789, 2784, 2793, 2793, 2792, 2795, 2798, 2774, 2794, 2795, 2797, 2785, 2794, 2797, 2796, 2795, 2799, 2791, 2798, 2793, 2799, 2794, 2793, 2793, 2792, 2790, 2793, 2794, 2796, 2791, 2793, 2796, 2793, 2795, 2792, 2798, 2796, 2797, 2795, 2792, 2790, 2794, 2796, 2793, 2797, 2791, 2794, 2796, 2793, 2790, 2785, 2796, 2796, 2794, 2797, 2795, 2799, 2788, 2790, 2793, 2796, 2796, 2795, 2795, 2796, 2797, 2798, 2796, 2791, 2794, 2794, 2796, 2794, 2796, 2799, 2798, 2795, 2795, 2797, 2796, 2792, 2797, 2795, 2795, 2797, 2799, 2798, 2796, 2799, 2797, 2797, 2796, 2799, 2796, 2796, 2793, 2792, 2794, 2794, 2797, 2794, 2799, 2796, 2796, 2800, 2796, 2794, 2794, 2800, 2796, 2796, 2799, 2799, 2794, 2791, 2793, 2798, 2792, 2769, 2795, 2791, 2800, 2795, 2796, 2798, 2790, 2798, 2797, 2795, 2794, 2797, 2796, 2800, 2798, 2798, 2798, 2797, 2792, 2798, 2797, 2796, 2798, 2800, 2799, 2798, 2796, 2799, 2796, 2798, 2799, 2798, 2797, 2799, 2794, 2800, 2793, 2800, 2799, 2799, 2799, 2799, 2797, 2798, 2798, 2797, 2799, 2797, 2796, 2799, 2795, 2798, 2784, 2798, 2797, 2797, 2797, 2792, 2796, 2799, 2797, 2797, 2789, 2797, 2793, 2797, 2772, 2796, 2796, 2797, 2797, 2796, 2796, 2798, 2794, 2791, 2800, 2793, 2785, 2798, 2799, 2790, 2793, 2797, 2797, 2793, 2800, 2798, 2797, 2796, 2798, 2795, 2795, 2793, 2795, 2795, 2797, 2799, 2797, 2795, 2796, 2799, 2800, 2793, 2795, 2788, 2798, 2800, 2799, 2798, 2790, 2776, 2790, 2797, 2797, 2795, 2796, 2795, 2795, 2797, 2783, 2789, 2800, 2797, 2797, 2796, 2795, 2787, 2797, 2797, 2800, 2774, 2797, 2796, 2799, 2797, 2800, 2799, 2787, 2793, 2799, 2800, 2791, 2796, 2799, 2799, 2797, 2800, 2799, 2799, 2796, 2798, 2799, 2799, 2799, 2797, 2799, 2800, 2770, 2797, 2800, 2799, 2797, 2799, 2800, 2798, 2798, 2789, 2800, 2800, 2798, 2799, 2793, 2799, 2798, 2797, 2797, 2797, 2797, 2798, 2800, 2785, 2796, 2796, 2797, 2800, 2799, 2799, 2799, 2796, 2800, 2798, 2798, 2789, 2797, 2800, 2779, 2798, 2796, 2800, 2799, 2797, 2800, 2794]  # y
+        y_axis_data2 = [1127, 1292, 1282, 1353, 1307, 1278, 1217, 1274, 1320, 1279, 1286, 1277, 1316, 1260, 1299, 1327, 1240, 1349, 1330, 1348, 1292, 1380, 1367, 1370, 1317, 1351, 1461, 1361, 1366, 1365, 1431, 1467, 1417, 1437, 1414, 1498, 1498, 1501, 1469, 1532, 1507, 1591, 1562, 1528, 1572, 1650, 1621, 1615, 1640, 1649, 1692, 1709, 1753, 1722, 1748, 1784, 1729, 1799, 1805, 1809, 1875, 1822, 1959, 1921, 1820, 1824, 1880, 1974, 1975, 2025, 1982, 2013, 2017, 2059, 2118, 2130, 2086, 2054, 2041, 2102, 2178, 2181, 2186, 2225, 2213, 2205, 2303, 2259, 2282, 2321, 2322, 2347, 2308, 2288, 2359, 2349, 2359, 2357, 2369, 2340, 2329, 2406, 2374, 2377, 2403, 2382, 2407, 2469, 2453, 2453, 2478, 2439, 2475, 2492, 2514, 2538, 2507, 2566, 2536, 2522, 2537, 2555, 2541, 2477, 2642, 2548, 2613, 2600, 2617, 2601, 2642, 2595, 2613, 2607, 2612, 2613, 2621, 2601, 2616, 2621, 2650, 2575, 2565, 2570, 2539, 2562, 2553, 2591, 2525, 2512, 2511, 2590, 2509, 2533, 2624, 2418, 2543, 2461, 2627, 2544, 2539, 2543, 2570, 2580, 2493, 2590, 2502, 2544, 2525, 2505, 2517, 2567, 2533, 2456, 2557, 2519, 2566, 2615, 2516, 2568, 2536, 2550, 2583, 2491, 2599, 2575, 2547, 2436, 2514, 2610, 2563, 2531, 2515, 2572, 2429, 2563, 2586, 2496, 2489, 2530, 2492, 2363, 2401, 2462, 2398, 2419, 2477, 2339, 2323, 2358, 2499, 2506, 2408, 2563, 2461, 2513, 2270, 2265, 2339, 2239, 2527, 2417, 2511, 2409, 2288, 2396, 2480, 2376, 2439, 2591, 2563, 2300, 2520, 2577, 2477, 2469, 2320, 2271, 2413, 2560, 2620, 2597, 2573, 2367, 2465, 2457, 2347, 2494, 2376, 2613, 2561, 2507, 2455, 2428, 2489, 2463, 2639, 2574, 2560, 2492, 2586, 2591, 2434, 2618, 2636, 2506, 2606, 2491, 2661, 2503, 2618, 2454, 2481, 2412, 2543, 2629, 2493, 2562, 2576, 2616, 2536, 2506, 2573, 2548, 2576, 2543, 2505, 2461, 2537, 2636, 2569, 2652, 2577, 2520, 2345, 2478, 2618, 2599, 2535, 2571, 2512, 2584, 2642, 2626, 2526, 2621, 2449, 2620, 2583, 2495, 2635, 2629, 2648, 2612, 2681, 2681, 2605, 2473, 2533, 2537, 2601, 2522, 2580, 2592, 2584, 2675, 2588, 2569, 2540, 2659, 2545, 2608, 2534, 2646, 2620, 2608, 2663, 2595, 2641, 2608, 2594, 2623, 2621, 2588, 2525, 2516, 2624, 2515, 2663, 2586, 2670, 2610, 2626, 2601, 2652, 2663, 2639, 2585, 2585, 2621, 2658, 2659, 2662, 2614, 2660, 2606, 2655, 2654, 2621, 2630, 2709, 2631, 2627, 2610, 2652, 2698, 2685, 2704, 2673, 2576, 2579, 2686, 2694, 2662, 2656, 2606, 2687, 2693, 2703, 2623, 2698, 2698, 2714, 2753, 2745, 2685, 2678, 2674, 2718, 2675, 2723, 2637, 2719, 2735, 2571, 2685, 2689, 2621, 2722, 2728, 2662, 2690, 2721, 2694, 2754, 2720, 2756, 2705, 2755, 2748, 2709, 2661, 2718, 2765, 2697, 2730, 2734, 2737, 2737, 2719, 2723, 2755, 2754, 2708, 2677, 2754, 2669, 2757, 2738, 2752, 2732, 2714, 2667, 2717, 2675, 2701, 2698, 2782, 2729, 2745, 2717, 2692, 2745, 2699, 2689, 2692, 2697, 2710, 2658, 2673, 2711, 2699, 2751, 2742, 2698, 2707, 2726, 2633, 2655, 2742, 2706, 2715, 2773, 2746, 2730, 2756, 2779, 2781, 2778, 2772, 2784, 2786, 2770, 2759, 2783, 2789, 2794, 2792, 2787, 2788, 2795, 2795, 2788, 2796, 2792, 2783, 2786, 2793, 2781, 2795, 2690, 2750, 2727, 2733, 2737, 2724, 2735, 2740, 2717, 2706, 2754, 2723, 2768, 2771, 2762, 2699, 2720, 2745, 2723, 2723, 2754, 2797, 2778, 2749, 2758, 2796, 2704, 2760, 2785, 2749, 2779, 2747, 2797, 2781, 2742, 2796, 2747, 2771, 2796, 2775, 2738, 2704, 2745, 2799, 2775, 2706, 2798, 2738, 2717, 2792, 2760, 2745, 2794, 2781, 2791, 2744, 2736, 2772, 2798, 2703, 2737, 2799, 2751, 2773, 2728, 2734, 2699, 2733, 2732, 2754, 2767, 2762, 2799, 2726, 2695, 2730, 2726, 2749, 2716, 2723, 2737, 2733, 2748, 2742, 2744, 2791, 2730, 2790, 2747, 2704, 2771, 2722, 2700, 2722, 2751, 2798, 2796, 2740, 2763, 2732, 2747, 2796, 2770, 2795, 2781, 2756, 2703, 2718, 2758, 2773, 2734, 2798, 2740, 2792, 2791, 2717, 2789, 2738, 2796, 2780, 2798, 2795, 2703, 2732, 2762, 2769, 2760, 2766, 2782, 2704, 2796, 2798, 2800, 2775, 2741, 2799, 2797, 2795, 2799, 2794, 2792, 2746, 2752, 2799, 2800, 2749, 2769, 2800, 2798, 2773, 2799, 2800, 2800, 2798, 2758, 2797, 2799, 2800, 2800, 2799, 2800, 2800, 2799, 2799, 2800, 2799, 2737, 2799, 2800, 2798, 2767, 2754, 2773, 2798, 2800, 2800, 2748, 2799, 2800, 2750, 2724, 2709, 2722, 2724, 2739, 2730, 2727, 2760, 2753, 2771, 2736, 2745, 2721, 2800, 2800, 2797, 2763, 2799, 2790, 2784, 2767, 2749, 2799, 2800, 2800, 2800, 2772, 2794, 2777, 2767, 2800, 2800, 2800, 2800, 2800, 2799, 2798, 2800, 2800, 2798, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2799, 2800, 2799, 2799, 2799, 2799, 2800, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2798, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2796, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2779, 2800, 2800, 2800, 2800, 2800, 2800, 2783, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2794, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2783, 2799, 2800, 2800, 2796, 2800, 2798, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2798, 2800, 2800, 2800, 2795, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2793, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2797, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2792, 2800, 2800, 2800, 2800, 2785, 2779, 2798, 2800, 2767, 2787, 2800, 2769, 2790, 2751, 2714, 2763, 2800, 2786, 2790, 2785, 2799, 2800, 2783, 2749, 2800, 2790, 2767, 2789, 2782, 2754, 2800, 2790, 2763, 2789, 2800, 2794, 2798, 2758, 2786, 2799, 2791, 2800, 2765, 2780, 2792, 2753, 2800, 2766, 2800, 2787, 2800, 2800, 2768, 2797, 2799, 2787, 2800, 2792, 2800, 2799, 2770, 2800, 2800, 2800, 2792, 2763, 2800, 2765, 2726, 2755, 2742, 2764, 2740, 2752, 2751, 2762, 2781, 2779, 2780, 2730, 2768, 2728, 2702, 2728, 2800, 2750, 2739, 2761, 2763, 2800, 2800, 2763, 2782, 2772, 2761, 2800, 2783, 2791, 2798, 2790, 2800, 2779, 2783, 2726, 2704, 2745, 2744, 2800, 2779, 2800, 2800, 2785, 2760, 2797, 2767, 2787, 2769, 2768]
+        y3 = [1181, 1254, 1322, 1270, 1243, 1298, 1255, 1336, 1318, 1345, 1299, 1346, 1322, 1330, 1296, 1388, 1381, 1323, 1337, 1323, 1349, 1270, 1356, 1398, 1386, 1418, 1451, 1365, 1354, 1368, 1477, 1440, 1425, 1386, 1411, 1432, 1399, 1449, 1483, 1419, 1470, 1460, 1394, 1447, 1484, 1404, 1513, 1391, 1445, 1459, 1397, 1510, 1461, 1484, 1521, 1446, 1472, 1511, 1592, 1599, 1530, 1609, 1576, 1529, 1661, 1533, 1578, 1569, 1636, 1588, 1589, 1641, 1620, 1597, 1583, 1582, 1585, 1635, 1665, 1637, 1598, 1603, 1675, 1618, 1608, 1639, 1649, 1670, 1724, 1709, 1686, 1656, 1680, 1691, 1749, 1725, 1745, 1722, 1765, 1714, 1735, 1684, 1811, 1745, 1767, 1777, 1784, 1800, 1782, 1818, 1824, 1807, 1749, 1832, 1806, 1846, 1805, 1852, 1869, 1820, 1835, 1812, 1903, 1859, 1942, 1916, 1783, 1931, 1892, 1824, 1939, 1859, 1933, 1875, 1963, 1887, 1998, 1924, 1959, 1930, 1950, 2001, 2035, 1973, 1950, 2057, 1971, 1987, 1906, 1975, 1995, 2004, 2055, 2056, 2049, 2127, 2055, 2090, 2097, 2034, 2006, 2103, 2088, 2055, 2133, 2129, 2125, 2071, 2070, 2104, 2094, 2095, 2140, 2125, 2162, 2031, 2147, 2132, 2135, 2084, 2153, 2166, 2086, 2125, 2167, 2227, 2204, 2140, 2166, 2217, 2235, 2228, 2216, 2198, 2193, 2213, 2186, 2273, 2176, 2194, 2234, 2228, 2210, 2209, 2233, 2205, 2229, 2231, 2280, 2247, 2192, 2263, 2225, 2243, 2249, 2298, 2271, 2299, 2305, 2324, 2286, 2340, 2321, 2319, 2320, 2254, 2353, 2311, 2315, 2352, 2293, 2323, 2343, 2316, 2306, 2382, 2292, 2357, 2300, 2360, 2302, 2379, 2353, 2371, 2363, 2399, 2334, 2380, 2392, 2381, 2374, 2353, 2403, 2358, 2382, 2389, 2414, 2421, 2353, 2409, 2398, 2435, 2385, 2361, 2389, 2428, 2416, 2410, 2479, 2416, 2434, 2417, 2457, 2431, 2437, 2450, 2389, 2440, 2482, 2424, 2436, 2389, 2474, 2477, 2460, 2436, 2472, 2466, 2476, 2456, 2437, 2430, 2429, 2485, 2400, 2430, 2460, 2455, 2441, 2484, 2480, 2479, 2463, 2466, 2437, 2490, 2478, 2507, 2523, 2465, 2486, 2518, 2510, 2454, 2491, 2527, 2456, 2505, 2505, 2496, 2498, 2516, 2488, 2499, 2521, 2533, 2486, 2526, 2491, 2494, 2531, 2527, 2513, 2505, 2502, 2517, 2498, 2538, 2490, 2522, 2543, 2526, 2538, 2560, 2516, 2510, 2532, 2525, 2536, 2532, 2543, 2502, 2516, 2525, 2548, 2522, 2558, 2525, 2547, 2532, 2521, 2517, 2497, 2518, 2534, 2522, 2520, 2544, 2555, 2528, 2513, 2534, 2534, 2536, 2547, 2544, 2537, 2500, 2540, 2559, 2551, 2556, 2564, 2514, 2533, 2541, 2548, 2563, 2559, 2552, 2559, 2545, 2547, 2569, 2557, 2546, 2549, 2559, 2555, 2556, 2549, 2536, 2552, 2569, 2542, 2559, 2582, 2546, 2563, 2554, 2562, 2571, 2548, 2582, 2560, 2557, 2556, 2561, 2556, 2557, 2565, 2564, 2546, 2577, 2559, 2578, 2564, 2541, 2565, 2576, 2570, 2565, 2555, 2581, 2550, 2586, 2577, 2565, 2570, 2576, 2575, 2574, 2564, 2584, 2577, 2567, 2584, 2573, 2573, 2570, 2579, 2567, 2573, 2568, 2574, 2565, 2583, 2572, 2579, 2567, 2570, 2585, 2579, 2578, 2580, 2590, 2591, 2574, 2587, 2587, 2580, 2585, 2591, 2583, 2596, 2582, 2590, 2572, 2577, 2584, 2591, 2585, 2574, 2576, 2589, 2580, 2583, 2580, 2579, 2586, 2573, 2583, 2591, 2590, 2578, 2582, 2597, 2588, 2575, 2586, 2583, 2594, 2581, 2579, 2591, 2592, 2583, 2586, 2589, 2582, 2583, 2589, 2593, 2577, 2592, 2594, 2580, 2588, 2583, 2594, 2570, 2576, 2595, 2596, 2583, 2589, 2577, 2582, 2580, 2585, 2582, 2587, 2592, 2584, 2583, 2589, 2595, 2591, 2608, 2593, 2602, 2595, 2602, 2600, 2602, 2589, 2601, 2597, 2594, 2596, 2620, 2604, 2582, 2589, 2584, 2596, 2604, 2598, 2601, 2600, 2586, 2594, 2586, 2598, 2601, 2596, 2617, 2602, 2606, 2595, 2596, 2609, 2591, 2597, 2586, 2592, 2599, 2606, 2607, 2586, 2598, 2587, 2598, 2609, 2606, 2597, 2599, 2572, 2606, 2596, 2601, 2609, 2605, 2591, 2618, 2596, 2599, 2605, 2610, 2599, 2590, 2596, 2615, 2588, 2606, 2617, 2611, 2605, 2602, 2622, 2605, 2614, 2592, 2611, 2596, 2601, 2614, 2587, 2599, 2609, 2605, 2594, 2610, 2615, 2605, 2612, 2629, 2633, 2635, 2603, 2610, 2621, 2630, 2639, 2621, 2628, 2639, 2596, 2623, 2647, 2644, 2628, 2637, 2643, 2584, 2650, 2636, 2622, 2663, 2637, 2642, 2656, 2655, 2661, 2633, 2655, 2677, 2651, 2673, 2667, 2659, 2658, 2677, 2671, 2673, 2663, 2669, 2680, 2665, 2668, 2683, 2691, 2690, 2685, 2698, 2703, 2691, 2720, 2702, 2708, 2713, 2713, 2734, 2613, 2702, 2673, 2714, 2698, 2726, 2668, 2723, 2705, 2700, 2715, 2736, 2712, 2715, 2726, 2722, 2733, 2730, 2720, 2709, 2735, 2727, 2697, 2660, 2663, 2618, 2690, 2675, 2686, 2666, 2672, 2657, 2640, 2717, 2686, 2630, 2676, 2677, 2649, 2619, 2655, 2687, 2678, 2688, 2677, 2658, 2633, 2611, 2675, 2635, 2649, 2692, 2664, 2664, 2643, 2675, 2680, 2714, 2665, 2700, 2691, 2645, 2680, 2684, 2718, 2626, 2667, 2675, 2708, 2654, 2653, 2699, 2720, 2708, 2715, 2678, 2704, 2714, 2687, 2706, 2719, 2664, 2722, 2674, 2708, 2717, 2728, 2712, 2720, 2766, 2695, 2718, 2706, 2735, 2703, 2719, 2752, 2728, 2727, 2696, 2706, 2735, 2752, 2715, 2710, 2740, 2745, 2732, 2734, 2738, 2733, 2697, 2736, 2738, 2743, 2729, 2735, 2743, 2701, 2754, 2756, 2778, 2740, 2728, 2719, 2731, 2745, 2751, 2706, 2702, 2729, 2736, 2750, 2751, 2729, 2724, 2749, 2739, 2764, 2770, 2745, 2759, 2708, 2719, 2739, 2715, 2767, 2731, 2755, 2746, 2740, 2759, 2739, 2741, 2771, 2740, 2754, 2744, 2768, 2743, 2761, 2762, 2760, 2766, 2775, 2770, 2764, 2773, 2776, 2753, 2718, 2765, 2754, 2779, 2771, 2742, 2744, 2744, 2737, 2743, 2767, 2754, 2768, 2759, 2752, 2766, 2752, 2785, 2779, 2779, 2760, 2726, 2772, 2786, 2770, 2760, 2751, 2746, 2743, 2775, 2785, 2768, 2769, 2753, 2735, 2769, 2763, 2758, 2755, 2749, 2779, 2765, 2781, 2766, 2776, 2765, 2776, 2772, 2765, 2780, 2773, 2759, 2762, 2780, 2769, 2775, 2787, 2764, 2780, 2771, 2741, 2782, 2779, 2774, 2780, 2776, 2781, 2772, 2773, 2784, 2768, 2732, 2774, 2766, 2769, 2778, 2756, 2783, 2754, 2695, 2772, 2773, 2778, 2784, 2770, 2782, 2767, 2786, 2765, 2742, 2768, 2770, 2784, 2783, 2771, 2770, 2786, 2763, 2782, 2787, 2771, 2785, 2789, 2765, 2749, 2776, 2761, 2786, 2772, 2774, 2769, 2783, 2791, 2793, 2779, 2753, 2786, 2781, 2785, 2788, 2747, 2789, 2782, 2779, 2790, 2789, 2747, 2778, 2794, 2787, 2790, 2767, 2785, 2793, 2655, 2785, 2773, 2789, 2755, 2788, 2763, 2764, 2792, 2792, 2795, 2785, 2764]
+        for i in range(0, 1000):
+            x_axis_data.append(i)
+            y1.append(y_axis_data[i])
+            y2.append(y_axis_data2[i])
+
+        plt.plot(x_axis_data, y1, 'b--', alpha=1, linewidth=1,
+                 label='MAPPO')  # 'bo-'表示蓝色实线，数据点实心原点标注
+        plt.plot(x_axis_data, y2, 'r--', alpha=0.5, linewidth=1, label='TMAPPO-2')
+        plt.plot(x_axis_data, y3, 'y--', alpha=0.5, linewidth=1, label='TMAPPO')
+
+        ## plot中参数的含义分别是横轴值，纵轴值，线的形状（'s'方块,'o'实心圆点，'*'五角星   ...，颜色，透明度,线的宽度和标签 ，
+
+
+        plt.legend()  # 显示上面的label
+        plt.xlabel('Epi')  # x_label
+        plt.ylabel('Transmit')  # y_label
+
+        # plt.ylim(-1,1)#仅设置y轴坐标范围
+        plt.show()
+
+    def plotDemo(self):
+        x_axis_data = [ ]  # x
+        y1 = []
+        y2= []
+        y_axis_data1 = [1136, 1273, 1265, 1262, 1316, 1315, 1286, 1231, 1274, 1380, 1361, 1316, 1334, 1325, 1338, 1312, 1313, 1335, 1317, 1345, 1337, 1360, 1392, 1287, 1449, 1415, 1396, 1430, 1443, 1370, 1436, 1399, 1406, 1492, 1434, 1505, 1505, 1521, 1543, 1595, 1528, 1588, 1528, 1520, 1597, 1615, 1562, 1650, 1702, 1670, 1673, 1743, 1702, 1766, 1737, 1740, 1753, 1738, 1755, 1750, 1786, 1831, 1817, 1883, 1885, 1877, 1843, 1911, 1924, 1945, 2013, 1963, 2010, 2042, 1994, 2058, 2080, 2079, 2112, 2087, 2118, 2085, 2220, 2200, 2195, 2218, 2241, 2245, 2252, 2331, 2301, 2354, 2298, 2388, 2372, 2375, 2377, 2435, 2328, 2379, 2452, 2368, 2437, 2471, 2456, 2466, 2439, 2505, 2483, 2442, 2519, 2505, 2470, 2530, 2541, 2518, 2517, 2537, 2491, 2552, 2577, 2565, 2575, 2597, 2606, 2571, 2620, 2581, 2620, 2582, 2616, 2646, 2659, 2645, 2693, 2645, 2697, 2673, 2679, 2673, 2682, 2685, 2706, 2703, 2710, 2709, 2694, 2684, 2687, 2681, 2701, 2702, 2719, 2737, 2750, 2726, 2745, 2752, 2749, 2757, 2767, 2774, 2757, 2755, 2762, 2769, 2768, 2762, 2770, 2766, 2772, 2775, 2761, 2786, 2775, 2758, 2782, 2775, 2752, 2766, 2772, 2789, 2784, 2783, 2709, 2773, 2769, 2774, 2758, 2778, 2784, 2788, 2782, 2787, 2744, 2765, 2778, 2769, 2763, 2774, 2735, 2774, 2692, 2783, 2790, 2780, 2768, 2789, 2786, 2794, 2753, 2766, 2788, 2742, 2768, 2790, 2778, 2784, 2782, 2778, 2759, 2784, 2783, 2792, 2784, 2778, 2786, 2790, 2797, 2788, 2791, 2790, 2788, 2792, 2796, 2792, 2795, 2789, 2776, 2787, 2751, 2783, 2787, 2794, 2791, 2787, 2792, 2791, 2797, 2798, 2785, 2791, 2794, 2798, 2794, 2796, 2784, 2796, 2794, 2784, 2786, 2789, 2793, 2796, 2798, 2789, 2757, 2795, 2797, 2798, 2785, 2797, 2732, 2800, 2791, 2798, 2800, 2796, 2794, 2793, 2799, 2799, 2799, 2797, 2799, 2792, 2796, 2777, 2800, 2796, 2794, 2799, 2800, 2798, 2798, 2799, 2784, 2795, 2799, 2759, 2796, 2741, 2796, 2784, 2792, 2775, 2792, 2763, 2800, 2799, 2799, 2793, 2795, 2793, 2793, 2798, 2799, 2799, 2787, 2795, 2788, 2778, 2793, 2800, 2790, 2795, 2767, 2779, 2797, 2772, 2784, 2798, 2797, 2797, 2800, 2796, 2794, 2799, 2800, 2799, 2800, 2798, 2799, 2799, 2800, 2799, 2800, 2800, 2798, 2797, 2799, 2799, 2797, 2797, 2799, 2799, 2800, 2800, 2797, 2798, 2799, 2800, 2799, 2799, 2795, 2798, 2798, 2800, 2798, 2800, 2796, 2800, 2794, 2796, 2799, 2799, 2800, 2758, 2752, 2799, 2800, 2796, 2799, 2799, 2799, 2794, 2796, 2795, 2799, 2791, 2774, 2785, 2796, 2786, 2798, 2800, 2799, 2800, 2760, 2799, 2793, 2800, 2797, 2800, 2800, 2799, 2695, 2736, 2800, 2798, 2798, 2798, 2722, 2799, 2800, 2798, 2799, 2769, 2800, 2800, 2800, 2790, 2799, 2800, 2727, 2799, 2763, 2800, 2799, 2763, 2799, 2800, 2798, 2800, 2753, 2800, 2800, 2800, 2732, 2800, 2800, 2799, 2799, 2800, 2799, 2800, 2799, 2799, 2800, 2800, 2800, 2800, 2800, 2733, 2800, 2798, 2799, 2800, 2799, 2800, 2798, 2798, 2800, 2799, 2800, 2799, 2800, 2798, 2800, 2800, 2800, 2800, 2797, 2798, 2800, 2730, 2800, 2799, 2798, 2798, 2800, 2800, 2763, 2781, 2798, 2799, 2799, 2696, 2799, 2784, 2799, 2799, 2799, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2737, 2800, 2686, 2800, 2800, 2800, 2799, 2800, 2796, 2800, 2800, 2781, 2800, 2800, 2800, 2799, 2798, 2800, 2798, 2780, 2785, 2756, 2800, 2800, 2800, 2799, 2780, 2800, 2714, 2800, 2800, 2675, 2799, 2793, 2650, 2799, 2800, 2800, 2800, 2799, 2800, 2646, 2725, 2798, 2799, 2615, 2791, 2799, 2790, 2799, 2800, 2788, 2800, 2800, 2800, 2800, 2800, 2799, 2796, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2797, 2800, 2800, 2800, 2799, 2800, 2800, 2787, 2800, 2758, 2799, 2791, 2797, 2658, 2719, 2800, 2800, 2724, 2800, 2799, 2784, 2796, 2743, 2800, 2798, 2795, 2800, 2800, 2800, 2799, 2800, 2800, 2752, 2771, 2800, 2717, 2800, 2721, 2749, 2654, 2621, 2799, 2691, 2639, 2579, 2661, 2731, 2778, 2799, 2723, 2722, 2650, 2661, 2733, 2799, 2585, 2603, 2711, 2659, 2679, 2696, 2634, 2592, 2644, 2800, 2740, 2755, 2660, 2561, 2681, 2680, 2698, 2739, 2503, 2800, 2663, 2622, 2724, 2696, 2688, 2707, 2780, 2631, 2712, 2634, 2649, 2691, 2713, 2738, 2690, 2546, 2715, 2745, 2625, 2621, 2608, 2645, 2788, 2661, 2750, 2620, 2800, 2649, 2675, 2776, 2702, 2713, 2793, 2741, 2673, 2790, 2788, 2750, 2761, 2748, 2799, 2798, 2798, 2790, 2800, 2784, 2797, 2787, 2795, 2784, 2800, 2787, 2792, 2796, 2799, 2795, 2792, 2796, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2775, 2800, 2796, 2779, 2800, 2716, 2660, 2656, 2753, 2800, 2792, 2697, 2689, 2736, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2778, 2800, 2771, 2740, 2800, 2695, 2734, 2800, 2772, 2792, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2787, 2793, 2800, 2800, 2786, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2777, 2800, 2800, 2800, 2800, 2793, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2788, 2800, 2782, 2800, 2800, 2800, 2797, 2774, 2750, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2775, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2793, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2756, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2778, 2800, 2800, 2800, 2758, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2729, 2800, 2800, 2800, 2800, 2800, 2780, 2755, 2767, 2800, 2800, 2749, 2800, 2800, 2800, 2795, 2799, 2794, 2748, 2773, 2771, 2753, 2746, 2797, 2791, 2732, 2800, 2794, 2800, 2800, 2800, 2800, 2792, 2800, 2800, 2800, 2800]
+        y_axis_data2 = [1127, 1292, 1282, 1353, 1307, 1278, 1217, 1274, 1320, 1279, 1286, 1277, 1316, 1260, 1299, 1327, 1240, 1349, 1330, 1348, 1292, 1380, 1367, 1370, 1317, 1351, 1461, 1361, 1366, 1365, 1431, 1467, 1417, 1437, 1414, 1498, 1498, 1501, 1469, 1532, 1507, 1591, 1562, 1528, 1572, 1650, 1621, 1615, 1640, 1649, 1692, 1709, 1753, 1722, 1748, 1784, 1729, 1799, 1805, 1809, 1875, 1822, 1959, 1921, 1820, 1824, 1880, 1974, 1975, 2025, 1982, 2013, 2017, 2059, 2118, 2130, 2086, 2054, 2041, 2102, 2178, 2181, 2186, 2225, 2213, 2205, 2303, 2259, 2282, 2321, 2322, 2347, 2308, 2288, 2359, 2349, 2359, 2357, 2369, 2340, 2329, 2406, 2374, 2377, 2403, 2382, 2407, 2469, 2453, 2453, 2478, 2439, 2475, 2492, 2514, 2538, 2507, 2566, 2536, 2522, 2537, 2555, 2541, 2477, 2642, 2548, 2613, 2600, 2617, 2601, 2642, 2595, 2613, 2607, 2612, 2613, 2621, 2601, 2616, 2621, 2650, 2575, 2565, 2570, 2539, 2562, 2553, 2591, 2525, 2512, 2511, 2590, 2509, 2533, 2624, 2418, 2543, 2461, 2627, 2544, 2539, 2543, 2570, 2580, 2493, 2590, 2502, 2544, 2525, 2505, 2517, 2567, 2533, 2456, 2557, 2519, 2566, 2615, 2516, 2568, 2536, 2550, 2583, 2491, 2599, 2575, 2547, 2436, 2514, 2610, 2563, 2531, 2515, 2572, 2429, 2563, 2586, 2496, 2489, 2530, 2492, 2363, 2401, 2462, 2398, 2419, 2477, 2339, 2323, 2358, 2499, 2506, 2408, 2563, 2461, 2513, 2270, 2265, 2339, 2239, 2527, 2417, 2511, 2409, 2288, 2396, 2480, 2376, 2439, 2591, 2563, 2300, 2520, 2577, 2477, 2469, 2320, 2271, 2413, 2560, 2620, 2597, 2573, 2367, 2465, 2457, 2347, 2494, 2376, 2613, 2561, 2507, 2455, 2428, 2489, 2463, 2639, 2574, 2560, 2492, 2586, 2591, 2434, 2618, 2636, 2506, 2606, 2491, 2661, 2503, 2618, 2454, 2481, 2412, 2543, 2629, 2493, 2562, 2576, 2616, 2536, 2506, 2573, 2548, 2576, 2543, 2505, 2461, 2537, 2636, 2569, 2652, 2577, 2520, 2345, 2478, 2618, 2599, 2535, 2571, 2512, 2584, 2642, 2626, 2526, 2621, 2449, 2620, 2583, 2495, 2635, 2629, 2648, 2612, 2681, 2681, 2605, 2473, 2533, 2537, 2601, 2522, 2580, 2592, 2584, 2675, 2588, 2569, 2540, 2659, 2545, 2608, 2534, 2646, 2620, 2608, 2663, 2595, 2641, 2608, 2594, 2623, 2621, 2588, 2525, 2516, 2624, 2515, 2663, 2586, 2670, 2610, 2626, 2601, 2652, 2663, 2639, 2585, 2585, 2621, 2658, 2659, 2662, 2614, 2660, 2606, 2655, 2654, 2621, 2630, 2709, 2631, 2627, 2610, 2652, 2698, 2685, 2704, 2673, 2576, 2579, 2686, 2694, 2662, 2656, 2606, 2687, 2693, 2703, 2623, 2698, 2698, 2714, 2753, 2745, 2685, 2678, 2674, 2718, 2675, 2723, 2637, 2719, 2735, 2571, 2685, 2689, 2621, 2722, 2728, 2662, 2690, 2721, 2694, 2754, 2720, 2756, 2705, 2755, 2748, 2709, 2661, 2718, 2765, 2697, 2730, 2734, 2737, 2737, 2719, 2723, 2755, 2754, 2708, 2677, 2754, 2669, 2757, 2738, 2752, 2732, 2714, 2667, 2717, 2675, 2701, 2698, 2782, 2729, 2745, 2717, 2692, 2745, 2699, 2689, 2692, 2697, 2710, 2658, 2673, 2711, 2699, 2751, 2742, 2698, 2707, 2726, 2633, 2655, 2742, 2706, 2715, 2773, 2746, 2730, 2756, 2779, 2781, 2778, 2772, 2784, 2786, 2770, 2759, 2783, 2789, 2794, 2792, 2787, 2788, 2795, 2795, 2788, 2796, 2792, 2783, 2786, 2793, 2781, 2795, 2690, 2750, 2727, 2733, 2737, 2724, 2735, 2740, 2717, 2706, 2754, 2723, 2768, 2771, 2762, 2699, 2720, 2745, 2723, 2723, 2754, 2797, 2778, 2749, 2758, 2796, 2704, 2760, 2785, 2749, 2779, 2747, 2797, 2781, 2742, 2796, 2747, 2771, 2796, 2775, 2738, 2704, 2745, 2799, 2775, 2706, 2798, 2738, 2717, 2792, 2760, 2745, 2794, 2781, 2791, 2744, 2736, 2772, 2798, 2703, 2737, 2799, 2751, 2773, 2728, 2734, 2699, 2733, 2732, 2754, 2767, 2762, 2799, 2726, 2695, 2730, 2726, 2749, 2716, 2723, 2737, 2733, 2748, 2742, 2744, 2791, 2730, 2790, 2747, 2704, 2771, 2722, 2700, 2722, 2751, 2798, 2796, 2740, 2763, 2732, 2747, 2796, 2770, 2795, 2781, 2756, 2703, 2718, 2758, 2773, 2734, 2798, 2740, 2792, 2791, 2717, 2789, 2738, 2796, 2780, 2798, 2795, 2703, 2732, 2762, 2769, 2760, 2766, 2782, 2704, 2796, 2798, 2800, 2775, 2741, 2799, 2797, 2795, 2799, 2794, 2792, 2746, 2752, 2799, 2800, 2749, 2769, 2800, 2798, 2773, 2799, 2800, 2800, 2798, 2758, 2797, 2799, 2800, 2800, 2799, 2800, 2800, 2799, 2799, 2800, 2799, 2737, 2799, 2800, 2798, 2767, 2754, 2773, 2798, 2800, 2800, 2748, 2799, 2800, 2750, 2724, 2709, 2722, 2724, 2739, 2730, 2727, 2760, 2753, 2771, 2736, 2745, 2721, 2800, 2800, 2797, 2763, 2799, 2790, 2784, 2767, 2749, 2799, 2800, 2800, 2800, 2772, 2794, 2777, 2767, 2800, 2800, 2800, 2800, 2800, 2799, 2798, 2800, 2800, 2798, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2799, 2800, 2799, 2799, 2799, 2799, 2800, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2798, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2796, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2779, 2800, 2800, 2800, 2800, 2800, 2800, 2783, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2794, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2783, 2799, 2800, 2800, 2796, 2800, 2798, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2798, 2800, 2800, 2800, 2795, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2793, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2797, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2792, 2800, 2800, 2800, 2800, 2785, 2779, 2798, 2800, 2767, 2787, 2800, 2769, 2790, 2751, 2714, 2763, 2800, 2786, 2790, 2785, 2799, 2800, 2783, 2749, 2800, 2790, 2767, 2789, 2782, 2754, 2800, 2790, 2763, 2789, 2800, 2794, 2798, 2758, 2786, 2799, 2791, 2800, 2765, 2780, 2792, 2753, 2800, 2766, 2800, 2787, 2800, 2800, 2768, 2797, 2799, 2787, 2800, 2792, 2800, 2799, 2770, 2800, 2800, 2800, 2792, 2763, 2800, 2765, 2726, 2755, 2742, 2764, 2740, 2752, 2751, 2762, 2781, 2779, 2780, 2730, 2768, 2728, 2702, 2728, 2800, 2750, 2739, 2761, 2763, 2800, 2800, 2763, 2782, 2772, 2761, 2800, 2783, 2791, 2798, 2790, 2800, 2779, 2783, 2726, 2704, 2745, 2744, 2800, 2779, 2800, 2800, 2785, 2760, 2797, 2767, 2787, 2769, 2768]
+        y4 = [1094, 1319, 1341, 1310, 1313, 1357, 1323, 1314, 1289, 1350, 1400, 1308, 1291, 1305, 1383, 1368, 1370, 1261, 1337, 1404, 1348, 1375, 1443, 1337, 1498, 1370, 1360, 1484, 1422, 1431, 1462, 1455, 1432, 1486, 1461, 1430, 1461, 1556, 1522, 1552, 1541, 1531, 1563, 1531, 1582, 1552, 1610, 1594, 1641, 1574, 1614, 1708, 1737, 1742, 1614, 1757, 1671, 1725, 1749, 1841, 1801, 1853, 1852, 1938, 1920, 1915, 1881, 1966, 2004, 1935, 1959, 2054, 2002, 2018, 2039, 2109, 2153, 2100, 2040, 2156, 2174, 2149, 2144, 2250, 2267, 2197, 2213, 2241, 2290, 2326, 2277, 2323, 2319, 2278, 2372, 2323, 2342, 2337, 2363, 2355, 2319, 2344, 2409, 2336, 2390, 2395, 2429, 2379, 2456, 2491, 2391, 2483, 2512, 2455, 2537, 2561, 2557, 2562, 2496, 2509, 2516, 2528, 2499, 2511, 2508, 2486, 2565, 2561, 2522, 2554, 2583, 2515, 2571, 2603, 2550, 2476, 2590, 2491, 2590, 2562, 2540, 2636, 2584, 2672, 2627, 2668, 2657, 2649, 2662, 2663, 2694, 2674, 2648, 2691, 2703, 2713, 2693, 2700, 2732, 2694, 2719, 2703, 2723, 2662, 2704, 2659, 2737, 2726, 2737, 2746, 2751, 2738, 2747, 2752, 2745, 2759, 2768, 2752, 2723, 2746, 2705, 2757, 2764, 2741, 2707, 2768, 2777, 2780, 2767, 2777, 2770, 2768, 2753, 2760, 2747, 2727, 2750, 2785, 2756, 2772, 2768, 2785, 2754, 2777, 2761, 2779, 2787, 2704, 2763, 2753, 2761, 2749, 2781, 2755, 2781, 2729, 2762, 2717, 2704, 2713, 2788, 2755, 2782, 2750, 2753, 2763, 2765, 2759, 2706, 2786, 2781, 2697, 2776, 2788, 2775, 2758, 2778, 2774, 2785, 2789, 2791, 2779, 2785, 2793, 2780, 2791, 2775, 2795, 2789, 2791, 2793, 2794, 2792, 2781, 2779, 2793, 2794, 2779, 2754, 2794, 2787, 2779, 2792, 2794, 2789, 2783, 2775, 2768, 2791, 2796, 2793, 2797, 2777, 2791, 2784, 2789, 2784, 2796, 2791, 2793, 2797, 2738, 2739, 2768, 2777, 2798, 2790, 2792, 2782, 2792, 2741, 2798, 2799, 2793, 2798, 2792, 2777, 2791, 2757, 2797, 2770, 2798, 2787, 2796, 2800, 2795, 2795, 2797, 2797, 2797, 2793, 2766, 2795, 2796, 2798, 2797, 2798, 2795, 2789, 2796, 2798, 2794, 2799, 2790, 2791, 2769, 2769, 2791, 2799, 2761, 2797, 2752, 2796, 2794, 2796, 2790, 2780, 2746, 2779, 2786, 2792, 2797, 2796, 2791, 2794, 2799, 2794, 2787, 2791, 2798, 2756, 2793, 2786, 2754, 2794, 2683, 2800, 2751, 2779, 2799, 2704, 2734, 2789, 2761, 2787, 2768, 2795, 2789, 2794, 2796, 2795, 2774, 2792, 2724, 2798, 2742, 2734, 2796, 2798, 2784, 2799, 2787, 2792, 2799, 2798, 2797, 2787, 2770, 2790, 2798, 2754, 2781, 2769, 2771, 2790, 2781, 2795, 2791, 2796, 2703, 2792, 2745, 2788, 2786, 2786, 2796, 2796, 2770, 2796, 2785, 2758, 2797, 2756, 2783, 2791, 2745, 2755, 2761, 2763, 2798, 2799, 2788, 2787, 2790, 2723, 2793, 2795, 2793, 2789, 2793, 2794, 2736, 2788, 2791, 2800, 2788, 2800, 2791, 2788, 2800, 2794, 2796, 2799, 2800, 2799, 2793, 2798, 2799, 2764, 2793, 2779, 2799, 2794, 2800, 2747, 2760, 2739, 2728, 2701, 2786, 2733, 2723, 2727, 2759, 2727, 2767, 2761, 2751, 2699, 2749, 2735, 2751, 2748, 2733, 2777, 2774, 2767, 2800, 2761, 2778, 2755, 2761, 2796, 2784, 2773, 2799, 2768, 2768, 2787, 2798, 2757, 2761, 2770, 2761, 2782, 2733, 2742, 2798, 2796, 2764, 2789, 2779, 2779, 2783, 2776, 2770, 2771, 2774, 2772, 2762, 2774, 2794, 2734, 2765, 2762, 2756, 2764, 2796, 2782, 2800, 2790, 2796, 2796, 2799, 2774, 2795, 2792, 2800, 2794, 2799, 2796, 2796, 2796, 2798, 2799, 2798, 2799, 2799, 2799, 2785, 2800, 2800, 2795, 2799, 2799, 2799, 2796, 2761, 2798, 2799, 2800, 2775, 2798, 2797, 2800, 2796, 2797, 2799, 2795, 2800, 2797, 2798, 2796, 2799, 2796, 2797, 2799, 2797, 2800, 2799, 2799, 2799, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2799, 2799, 2800, 2800, 2799, 2800, 2797, 2800, 2800, 2800, 2800, 2799, 2799, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2765, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2799, 2799, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2796, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2799, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2795, 2800, 2800, 2800, 2799, 2800, 2800, 2793, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2799, 2799, 2800, 2800, 2800, 2800, 2795, 2800, 2800, 2791, 2792, 2799, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2794, 2800, 2799, 2799, 2785, 2798, 2800, 2798, 2800, 2800, 2790, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2794, 2800, 2800, 2800, 2799, 2794, 2787, 2800, 2797, 2791, 2800, 2799, 2798, 2750, 2786, 2800, 2769, 2797, 2800, 2797, 2793, 2798, 2795, 2770, 2800, 2794, 2800, 2751, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2759, 2798, 2797, 2800, 2800, 2797, 2798, 2800, 2798, 2800, 2800, 2799, 2800, 2800, 2799, 2800, 2800, 2798, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2777, 2800, 2800, 2800, 2800, 2800, 2800, 2785, 2799, 2790, 2800, 2800, 2800, 2768, 2769, 2800, 2766, 2772, 2799, 2799, 2800, 2797, 2799, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2791, 2800, 2800, 2800, 2800, 2796, 2800, 2800, 2800, 2732, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2734, 2800, 2796, 2786, 2800, 2791, 2800, 2798, 2621, 2800, 2799, 2800, 2800, 2797, 2651, 2800, 2744, 2800, 2800, 2800, 2798, 2800, 2800, 2797, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2738, 2800, 2793, 2800, 2800, 2732, 2791, 2800, 2800, 2732, 2800, 2800, 2644, 2800, 2800, 2757, 2797, 2800, 2799, 2800, 2799, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2781, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2735, 2799, 2653, 2799, 2800, 2747, 2777, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2799, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800, 2800]
+
+        for i in range(0, 1000):
+            x_axis_data.append(i)
+            y1.append(y_axis_data1[i])
+            y2.append(y_axis_data2[i])
+
+        plt.plot(x_axis_data, y1, 'b--', alpha=1, linewidth=1,
+                 label='MAPPO-t1')  # 'bo-'表示蓝色实线，数据点实心原点标注
+        plt.plot(x_axis_data, y2, 'r--', alpha=0.5, linewidth=1, label='TMAPPO-t2')
+        plt.plot(x_axis_data, y4, 'y--', alpha=0.5, linewidth=1, label='TMAPPO-t4')
+
+        ## plot中参数的含义分别是横轴值，纵轴值，线的形状（'s'方块,'o'实心圆点，'*'五角星   ...，颜色，透明度,线的宽度和标签 ，
+
+
+        plt.legend()  # 显示上面的label
+        plt.xlabel('Epi')  # x_label
+        plt.ylabel('Transmit')  # y_label
+
+        # plt.ylim(-1,1)#仅设置y轴坐标范围
+        plt.show()
+
+p = plot()
+p.plotDemo()
+
+transformer_model = nn.Transformer(nhead=16, num_encoder_layers=12)
+src = torch.rand((10, 32, 512))
+tgt = torch.rand((10, 32, 512))
+out = transformer_model(src, tgt)
+print(out)
\ No newline at end of file

runner/separated/env_runner.py

@@ -26,7 +26,6 @@ class EnvRunner(Runner):
             if self.use_linear_lr_decay:
                 for agent_id in range(self.num_agents):
                     self.trainer[agent_id].policy.lr_decay(episode, episodes)
-
             for step in range(self.episode_length):
                 # Sample actions
                 (
@@ -63,15 +62,17 @@ class EnvRunner(Runner):
             # post process
             total_num_steps = (episode + 1) * self.episode_length * self.n_rollout_threads
 
+            self.envs.reset()
+
             # save model
-            if episode % self.save_interval == 0 or episode == episodes - 1:
-                self.save()
+            #if episode % self.save_interval == 0 or episode == episodes - 1:
+#                self.save()
 
             # log information
             if episode % self.log_interval == 0:
                 end = time.time()
                 print(
-                    "\n Scenario {} Algo {} Exp {} updates {}/{} episodes, total num timesteps {}/{}, FPS {}.\n".format(
+                    "\n Scenario2 {} Algo {} Exp {} updates {}/{} episodes, total num timesteps {}/{}, FPS {}.\n".format(
                         self.all_args.scenario_name,
                         self.algorithm_name,
                         self.experiment_name,
@@ -111,6 +112,7 @@ class EnvRunner(Runner):
             share_obs.append(list(chain(*o)))
         share_obs = np.array(share_obs)  # shape = [env_num, agent_num * obs_dim]
 
+
         for agent_id in range(self.num_agents):
             if not self.use_centralized_V:
                 share_obs = np.array(list(obs[:, agent_id]))
@@ -137,6 +139,7 @@ class EnvRunner(Runner):
                 self.buffer[agent_id].rnn_states_critic[step],
                 self.buffer[agent_id].masks[step],
             )
+
             # [agents, envs, dim]
             values.append(_t2n(value))
             action = _t2n(action)
@@ -171,8 +174,8 @@ class EnvRunner(Runner):
             actions_env.append(one_hot_action_env)
 
         values = np.array(values).transpose(1, 0, 2)
-        actions = np.array(actions).transpose(1, 0, 2)
-        action_log_probs = np.array(action_log_probs).transpose(1, 0, 2)
+#        actions = np.array(actions).transpose(1, 0, 2)
+  #      action_log_probs = np.array(action_log_probs).transpose(1, 0, 2)
         rnn_states = np.array(rnn_states).transpose(1, 0, 2, 3)
         rnn_states_critic = np.array(rnn_states_critic).transpose(1, 0, 2, 3)
 
@@ -223,8 +226,8 @@ class EnvRunner(Runner):
                 np.array(list(obs[:, agent_id])),
                 rnn_states[:, agent_id],
                 rnn_states_critic[:, agent_id],
-                actions[:, agent_id],
-                action_log_probs[:, agent_id],
+                actions[agent_id],
+                action_log_probs[agent_id],
                 values[:, agent_id],
                 rewards[:, agent_id],
                 masks[:, agent_id],

runner/separated/env_runner2.py

+import time
+from itertools import chain
+
+import imageio
+import matplotlib.pyplot as plt
+import numpy as np
+import torch
+
+from MAPPO.runner.separated.base_runner import Runner
+
+
+def _t2n(x):
+    return x.detach().cpu().numpy()
+
+
+class EnvRunner(Runner):
+    def __init__(self, config):
+        super(EnvRunner, self).__init__(config)
+        self.init_obs = None
+
+    def run(self):
+        self.warmup()
+
+        max_reward = 0
+        association_reward_log = [[] for i in range(self.n_rollout_threads)]
+        max_thread_num = 0
+        max_episode = 0
+        start = time.time()
+        episodes = int(self.num_env_steps) // self.episode_length // self.n_rollout_threads
+
+        max_obs_log = []
+
+        for episode in range(episodes):
+            if self.use_linear_lr_decay:
+                for agent_id in range(self.num_agents):
+                    self.trainer[agent_id].policy.lr_decay(episode, episodes)
+
+            association_reward_step = []
+            obs_log = [self.init_obs]
+            reward_log = [0 for i in range(self.n_rollout_threads)]
+
+
+            for step in range(self.episode_length):
+                # Sample actions
+                (
+                    values,
+                    actions,
+                    action_log_probs,
+                    rnn_states,
+                    rnn_states_critic,
+                    actions_env,
+                ) = self.collect(step)
+
+                # Obser reward and next obs
+                obs, rewards, dones, infos = self.envs.step(actions_env)
+                obs_log.append(obs)
+
+                data = (
+                    obs,
+                    rewards,
+                    dones,
+                    infos,
+                    values,
+                    actions,
+                    action_log_probs,
+                    rnn_states,
+                    rnn_states_critic,
+                )
+
+                # insert data into buffer
+                self.insert(data)
+
+                for i in range(self.n_rollout_threads):
+                    reward_log[i] += rewards[i][0]
+
+                association_reward_step.append(rewards[:, 0])
+
+            # compute return and update network
+            self.compute()
+            train_infos = self.train()
+
+            # post process
+            total_num_steps = (episode + 1) * self.episode_length * self.n_rollout_threads
+
+            self.envs.reset()
+
+            # save model
+            if episode % self.save_interval == 0 or episode == episodes - 1:
+                self.save()
+
+            for i in range(self.n_rollout_threads):
+                association_reward_log[i].append(np.sum(np.array(association_reward_step)[:, i]))
+                if reward_log[i] > max_reward:
+                    max_reward = reward_log[i]
+                    max_thread_num = i
+                    max_episode = episode
+                    max_obs_log = np.array(obs_log)[:,i,0]
+
+            # log information
+            if episode % self.log_interval == 0:
+                end = time.time()
+                print(
+                    "\n Scenario {} Algo {} Exp {} updates {}/{} episodes, total num timesteps {}/{}, FPS {}.\n".format(
+                        self.all_args.scenario_name,
+                        self.algorithm_name,
+                        self.experiment_name,
+                        episode,
+                        episodes,
+                        total_num_steps,
+                        self.num_env_steps,
+                        int(total_num_steps / (end - start)),
+                    )
+                )
+
+                print("\n rewards{}\n".format(rewards[:, 0]))
+
+                if self.env_name == "MPE":
+                    for agent_id in range(self.num_agents):
+                        idv_rews = []
+                        for info in infos:
+                            if "individual_reward" in info[agent_id].keys():
+                                idv_rews.append(info[agent_id]["individual_reward"])
+                        train_infos[agent_id].update({"individual_rewards": np.mean(idv_rews)})
+                        train_infos[agent_id].update(
+                            {
+                                "average_episode_rewards": np.mean(self.buffer[agent_id].rewards)
+                                                           * self.episode_length
+                            }
+                        )
+                self.log_train(train_infos, total_num_steps)
+
+            # eval
+            if episode % self.eval_interval == 0 and self.use_eval:
+                self.eval(total_num_steps)
+
+        print("thread:{},epi:{}".format(max_thread_num,max_episode))
+
+        uav_trace = [[],[],[],[]]
+        mu_pos = self.envs.envs[max_thread_num].env.init_mu_pos
+
+        for i in range(4):
+            for j in range(self.episode_length):
+                uav_trace[i].append([max_obs_log[j][i*2],max_obs_log[j][i*2+1]])
+
+        plt.figure(figsize=(10, 6))
+        plt.xlim(0, 1800)  # x轴的刻度范围
+        plt.ylim(0, 1800)  # y轴的刻度范围
+        plt.xlabel('经度', fontproperties="simhei")  # x轴的标题
+        plt.ylabel('纬度', fontproperties="simhei")  # y轴的标题
+        # 绘制各个点及点所代表地点名称
+
+        color = ['red','green','blue','purple']
+
+        for i in range(len(mu_pos)):
+            plt.plot(mu_pos[i][0],mu_pos[i][1],'o', color='black')
+            plt.text(mu_pos[i][0],mu_pos[i][1],str(i),fontsize=10, ha='right', va='bottom')
+
+        for i in range(len(uav_trace)):
+            for j in range(len(uav_trace[i])):
+                plt.plot(uav_trace[i][j][0], uav_trace[i][j][1], ',', color=color[i])
+        # 连接各个点
+            for j in range(len(uav_trace[i])-1):
+                start = (uav_trace[i][j][0], uav_trace[i][j+1][0])
+                end = (uav_trace[i][j][1], uav_trace[i][j+1][1])
+                plt.plot(start, end, color=color[i])
+
+        for i in range(len(uav_trace)):
+            plt.plot(uav_trace[i][0][0], uav_trace[i][0][1], '*', color='orange')
+            plt.plot(uav_trace[i][len(uav_trace[i])-1][0], uav_trace[i][len(uav_trace[i])-1][1], '*', color='pink')
+        plt.show()
+
+
+        for i in range(self.n_rollout_threads):
+            xs = list(range(0, len(association_reward_log[i])))
+            ys = association_reward_log[i]
+            plt.plot(xs, ys)
+
+            # 添加标题和坐标轴标签
+            plt.title("Square Curve")
+            plt.xlabel("x-axis")
+            plt.ylabel("y-axis")
+
+            # 显示图形
+            plt.show()
+
+    def warmup(self):
+        # reset env
+        obs = self.envs.reset()  # shape = [env_num, agent_num, obs_dim]
+        self.init_obs = obs.copy()
+
+        share_obs = []
+        for o in obs:
+            share_obs.append(list(chain(*o)))
+        share_obs = np.array(share_obs)  # shape = [env_num, agent_num * obs_dim]
+
+        for agent_id in range(self.num_agents):
+            if not self.use_centralized_V:
+                share_obs = np.array(list(obs[:, agent_id]))
+            self.buffer[agent_id].share_obs[0] = share_obs.copy()
+            self.buffer[agent_id].obs[0] = np.array(list(obs[:, agent_id])).copy()
+
+    @torch.no_grad()
+    def collect(self, step):
+        values = []
+        actions = []
+        temp_actions_env = []
+        action_log_probs = []
+        rnn_states = []
+        rnn_states_critic = []
+
+        for agent_id in range(self.num_agents):
+            self.trainer[agent_id].prep_rollout()
+            value, action, action_log_prob, rnn_state, rnn_state_critic = self.trainer[
+                agent_id
+            ].policy.get_actions(
+                self.buffer[agent_id].share_obs[step],
+                self.buffer[agent_id].obs[step],
+                self.buffer[agent_id].rnn_states[step],
+                self.buffer[agent_id].rnn_states_critic[step],
+                self.buffer[agent_id].masks[step],
+            )
+            # [agents, envs, dim]
+            values.append(_t2n(value))
+            action = _t2n(action)
+            # rearrange action
+            if self.envs.action_space[agent_id].__class__.__name__ == "MultiDiscrete":
+                for i in range(self.envs.action_space[agent_id].shape):
+                    uc_action_env = np.eye(self.envs.action_space[agent_id].high[i] + 1)[action[:, i]]
+                    if i == 0:
+                        action_env = uc_action_env
+                    else:
+                        action_env = np.concatenate((action_env, uc_action_env), axis=1)
+            elif self.envs.action_space[agent_id].__class__.__name__ == "Discrete":
+                action_env = np.squeeze(np.eye(self.envs.action_space[agent_id].n)[action], 1)
+            else:
+                # TODO 这里改造成自己环境需要的形式即可
+                # TODO Here, you can change the action_env to the form you need
+                for num in range(self.n_rollout_threads):
+                    act = action[num]
+                    act[0] = act[0] * 180
+                    act[1] = act[1] * 25
+                action_env = action
+                # raise NotImplementedError
+
+            actions.append(action)
+            temp_actions_env.append(action_env)
+            action_log_probs.append(_t2n(action_log_prob))
+            rnn_states.append(_t2n(rnn_state))
+            rnn_states_critic.append(_t2n(rnn_state_critic))
+
+        # [envs, agents, dim]
+        actions_env = []
+        for i in range(self.n_rollout_threads):
+            one_hot_action_env = []
+            for temp_action_env in temp_actions_env:
+                one_hot_action_env.append(temp_action_env[i])
+            actions_env.append(one_hot_action_env)
+
+        values = np.array(values).transpose(1, 0, 2)
+        # actions = np.array(actions).transpose(1, 0, 2)
+        # action_log_probs = np.array(action_log_probs).transpose(1, 0, 2)
+        rnn_states = np.array(rnn_states).transpose(1, 0, 2, 3)
+        rnn_states_critic = np.array(rnn_states_critic).transpose(1, 0, 2, 3)
+
+        return (
+            values,
+            actions,
+            action_log_probs,
+            rnn_states,
+            rnn_states_critic,
+            actions_env,
+        )
+
+    def insert(self, data):
+        (
+            obs,
+            rewards,
+            dones,
+            infos,
+            values,
+            actions,
+            action_log_probs,
+            rnn_states,
+            rnn_states_critic,
+        ) = data
+
+        rnn_states[dones == True] = np.zeros(
+            ((dones == True).sum(), self.recurrent_N, self.hidden_size),
+            dtype=np.float32,
+        )
+        rnn_states_critic[dones == True] = np.zeros(
+            ((dones == True).sum(), self.recurrent_N, self.hidden_size),
+            dtype=np.float32,
+        )
+        masks = np.ones((self.n_rollout_threads, self.num_agents, 1), dtype=np.float32)
+        masks[dones == True] = np.zeros(((dones == True).sum(), 1), dtype=np.float32)
+
+        share_obs = []
+        for o in obs:
+            share_obs.append(list(chain(*o)))
+        share_obs = np.array(share_obs)
+
+        for agent_id in range(self.num_agents):
+            if not self.use_centralized_V:
+                share_obs = np.array(list(obs[:, agent_id]))
+
+            self.buffer[agent_id].insert(
+                share_obs,
+                np.array(list(obs[:, agent_id])),
+                rnn_states[:, agent_id],
+                rnn_states_critic[:, agent_id],
+                actions[agent_id],
+                action_log_probs[agent_id],
+                values[:, agent_id],
+                rewards[:, agent_id],
+                masks[:, agent_id],
+            )
+
+    @torch.no_grad()
+    def eval(self, total_num_steps):
+        eval_episode_rewards = []
+        eval_obs = self.eval_envs.reset()
+
+        eval_rnn_states = np.zeros(
+            (
+                self.n_eval_rollout_threads,
+                self.num_agents,
+                self.recurrent_N,
+                self.hidden_size,
+            ),
+            dtype=np.float32,
+        )
+        eval_masks = np.ones((self.n_eval_rollout_threads, self.num_agents, 1), dtype=np.float32)
+
+        for eval_step in range(self.episode_length):
+            eval_temp_actions_env = []
+            for agent_id in range(self.num_agents):
+                self.trainer[agent_id].prep_rollout()
+                eval_action, eval_rnn_state = self.trainer[agent_id].policy.act(
+                    np.array(list(eval_obs[:, agent_id])),
+                    eval_rnn_states[:, agent_id],
+                    eval_masks[:, agent_id],
+                    deterministic=True,
+                )
+
+                eval_action = eval_action.detach().cpu().numpy()
+                # rearrange action
+                if self.eval_envs.action_space[agent_id].__class__.__name__ == "MultiDiscrete":
+                    for i in range(self.eval_envs.action_space[agent_id].shape):
+                        eval_uc_action_env = np.eye(self.eval_envs.action_space[agent_id].high[i] + 1)[
+                            eval_action[:, i]
+                        ]
+                        if i == 0:
+                            eval_action_env = eval_uc_action_env
+                        else:
+                            eval_action_env = np.concatenate((eval_action_env, eval_uc_action_env), axis=1)
+                elif self.eval_envs.action_space[agent_id].__class__.__name__ == "Discrete":
+                    eval_action_env = np.squeeze(
+                        np.eye(self.eval_envs.action_space[agent_id].n)[eval_action], 1
+                    )
+                else:
+                    raise NotImplementedError
+
+                eval_temp_actions_env.append(eval_action_env)
+                eval_rnn_states[:, agent_id] = _t2n(eval_rnn_state)
+
+            # [envs, agents, dim]
+            eval_actions_env = []
+            for i in range(self.n_eval_rollout_threads):
+                eval_one_hot_action_env = []
+                for eval_temp_action_env in eval_temp_actions_env:
+                    eval_one_hot_action_env.append(eval_temp_action_env[i])
+                eval_actions_env.append(eval_one_hot_action_env)
+
+            # Obser reward and next obs
+            eval_obs, eval_rewards, eval_dones, eval_infos = self.eval_envs.step(eval_actions_env)
+            eval_episode_rewards.append(eval_rewards)
+
+            eval_rnn_states[eval_dones == True] = np.zeros(
+                ((eval_dones == True).sum(), self.recurrent_N, self.hidden_size),
+                dtype=np.float32,
+            )
+            eval_masks = np.ones((self.n_eval_rollout_threads, self.num_agents, 1), dtype=np.float32)
+            eval_masks[eval_dones == True] = np.zeros(((eval_dones == True).sum(), 1), dtype=np.float32)
+
+        eval_episode_rewards = np.array(eval_episode_rewards)
+
+        eval_train_infos = []
+        for agent_id in range(self.num_agents):
+            eval_average_episode_rewards = np.mean(np.sum(eval_episode_rewards[:, :, agent_id], axis=0))
+            eval_train_infos.append({"eval_average_episode_rewards": eval_average_episode_rewards})
+            print("eval average episode rewards of agent%i: " % agent_id + str(eval_average_episode_rewards))
+
+        self.log_train(eval_train_infos, total_num_steps)
+
+    @torch.no_grad()
+    def render(self):
+        all_frames = []
+        for episode in range(self.all_args.render_episodes):
+            episode_rewards = []
+            obs = self.envs.reset()
+            if self.all_args.save_gifs:
+                image = self.envs.render("rgb_array")[0][0]
+                all_frames.append(image)
+
+            rnn_states = np.zeros(
+                (
+                    self.n_rollout_threads,
+                    self.num_agents,
+                    self.recurrent_N,
+                    self.hidden_size,
+                ),
+                dtype=np.float32,
+            )
+            masks = np.ones((self.n_rollout_threads, self.num_agents, 1), dtype=np.float32)
+
+            for step in range(self.episode_length):
+                calc_start = time.time()
+
+                temp_actions_env = []
+                for agent_id in range(self.num_agents):
+                    if not self.use_centralized_V:
+                        share_obs = np.array(list(obs[:, agent_id]))
+                    self.trainer[agent_id].prep_rollout()
+                    action, rnn_state = self.trainer[agent_id].policy.act(
+                        np.array(list(obs[:, agent_id])),
+                        rnn_states[:, agent_id],
+                        masks[:, agent_id],
+                        deterministic=True,
+                    )
+
+                    action = action.detach().cpu().numpy()
+                    # rearrange action
+                    if self.envs.action_space[agent_id].__class__.__name__ == "MultiDiscrete":
+                        for i in range(self.envs.action_space[agent_id].shape):
+                            uc_action_env = np.eye(self.envs.action_space[agent_id].high[i] + 1)[action[:, i]]
+                            if i == 0:
+                                action_env = uc_action_env
+                            else:
+                                action_env = np.concatenate((action_env, uc_action_env), axis=1)
+                    elif self.envs.action_space[agent_id].__class__.__name__ == "Discrete":
+                        action_env = np.squeeze(np.eye(self.envs.action_space[agent_id].n)[action], 1)
+                    else:
+                        raise NotImplementedError
+
+                    temp_actions_env.append(action_env)
+                    rnn_states[:, agent_id] = _t2n(rnn_state)
+
+                # [envs, agents, dim]
+                actions_env = []
+                for i in range(self.n_rollout_threads):
+                    one_hot_action_env = []
+                    for temp_action_env in temp_actions_env:
+                        one_hot_action_env.append(temp_action_env[i])
+                    actions_env.append(one_hot_action_env)
+
+                # Obser reward and next obs
+                obs, rewards, dones, infos = self.envs.step(actions_env)
+                episode_rewards.append(rewards)
+
+                rnn_states[dones == True] = np.zeros(
+                    ((dones == True).sum(), self.recurrent_N, self.hidden_size),
+                    dtype=np.float32,
+                )
+                masks = np.ones((self.n_rollout_threads, self.num_agents, 1), dtype=np.float32)
+                masks[dones == True] = np.zeros(((dones == True).sum(), 1), dtype=np.float32)
+
+                if self.all_args.save_gifs:
+                    image = self.envs.render("rgb_array")[0][0]
+                    all_frames.append(image)
+                    calc_end = time.time()
+                    elapsed = calc_end - calc_start
+                    if elapsed < self.all_args.ifi:
+                        time.sleep(self.all_args.ifi - elapsed)
+
+            episode_rewards = np.array(episode_rewards)
+            for agent_id in range(self.num_agents):
+                average_episode_rewards = np.mean(np.sum(episode_rewards[:, :, agent_id], axis=0))
+                print("eval average episode rewards of agent%i: " % agent_id + str(average_episode_rewards))
+
+        if self.all_args.save_gifs:
+            imageio.mimsave(
+                str(self.gif_dir) + "/render.gif",
+                all_frames,
+                duration=self.all_args.ifi,
+            )

runner/shared/env_runner.py

@@ -77,7 +77,7 @@ class EnvRunner(Runner):
             if episode % self.log_interval == 0:
                 end = time.time()
                 print(
-                    "\n Scenario {} Algo {} Exp {} updates {}/{} episodes, total num timesteps {}/{}, FPS {}.\n".format(
+                    "\n Sce2nario {} Algo {} Exp {} updates {}/{} episodes, total num timesteps {}/{}, FPS {}.\n".format(
                         self.all_args.scenario_name,
                         self.algorithm_name,
                         self.experiment_name,
@@ -88,17 +88,7 @@ class EnvRunner(Runner):
                         int(total_num_steps / (end - start)),
                     )
                 )
-
-                # if self.env_name == "MPE":
-                #     env_infos = {}
-                #     for agent_id in range(self.num_agents):
-                #         idv_rews = []
-                #         for info in infos:
-                #             if 'individual_reward' in info[agent_id].keys():
-                #                 idv_rews.append(info[agent_id]['individual_reward'])
-                #         agent_k = 'agent%i/individual_rewards' % agent_id
-                #         env_infos[agent_k] = idv_rews
-
+                print('11111')
                 train_infos["average_episode_rewards"] = np.mean(self.buffer.rewards) * self.episode_length
                 print("average episode rewards is {}".format(train_infos["average_episode_rewards"]))
                 self.log_train(train_infos, total_num_steps)

train/train.py

@@ -67,9 +67,9 @@ def make_eval_env(all_args):
 
 
 def parse_args(args, parser):
-    parser.add_argument("--scenario_name", type=str, default="MyEnv", help="Which scenario to run on")
+    parser.add_argument("--scenario_name", type=str, default="MPE", help="Which scenario to run on")
     parser.add_argument("--num_landmarks", type=int, default=3)
-    parser.add_argument("--num_agents", type=int, default=2, help="number of players")
+    parser.add_argument("--num_agents", type=int, default=9, help="number of players")
 
     all_args = parser.parse_known_args(args)[0]