model improved

maker.py

@@ -16,7 +16,7 @@ class UDEmbedsDataset(object):
     self.tokenizer=tokenizer
     self.embeddings=embeddings
     self.seeks=[0]
-    label=set(["SYM","SYM."])
+    label=set(["SYM","SYM.","SYM|_"])
     dep=set()
     s=self.conllu.readline()
     while s!="":
@@ -66,8 +66,9 @@ class UDEmbedsDataset(object):
       emb=self.embeddings
     else:
       import torch
-      if len(x)<128:
+      if len(x)<127:
         x=[True]*len(x)
+        w=(len(x)+2)*(len(x)+1)/2
       else:
         w=sum([len(x)-i+1 if b else 0 for i,b in enumerate(x)])+1
         for i in range(len(x)):
@@ -84,6 +85,11 @@ class UDEmbedsDataset(object):
           for j in range(i+1,len(x)):
             ids.append(j)
             upos.append(p[j]+"|"+d[j] if int(c[j][6])==i+1 else p[i]+"|"+d[i] if int(c[i][6])==j+1 else p[j]+"|_")
+          if w>8192 and i>0:
+            while w>8192 and upos[-1].endswith("|_"):
+              upos.pop(-1)
+              ids.pop(-1)
+              w-=1
           ids.append(-1)
           upos.append("SYM|_")
       with torch.no_grad():

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:844fe48fcb5c513dd8c954d8d4d546d4fa8f532babfa99dc4e2ec6feabce90b1
+oid sha256:3c73d3e7a178593b540a253d77587a8c8aaa459c1a4e621d5f20a40835a25785
 size 1392836610