generating data

data/create_data.py

@@ -0,0 +1,32 @@
+import random
+
+
+def main():
+    print('Creating data...')
+    # Create JSON data
+    # Create JSON data
+
+    WordList = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q']
+    file = open('data.json', 'w')
+    for _ in range(100):
+        file.write('{"translation":{"let":')
+        randomNum = random.randint(1, 50)
+        word = []
+        for i in random.sample(range(1, 100), randomNum):
+            word.append(WordList[i%17])
+        wordStr = ''.join(word)
+        wordRev = wordStr[::-1]
+        file.write(f'"{wordStr}", "rev" :"{wordRev}"')
+        file.write('}}\n')
+
+    file.close()
+
+    
+
+
+
+
+
+
+if __name__ == '__main__':
+    main()

data/data.json

@@ -0,0 +1,100 @@
+{"translation":{"let":"kmeihdamhqbppnknbgjmaibqhqiiemodjcecfqkofb", "rev" :"bfokqfcecjdomeiiqhqbiamjgbnknppbqhmadhiemk"}}
+{"translation":{"let":"feqcklngqpkljodgijbqkaanpeqnalcohjiaaoepbeblcigdl", "rev" :"ldgiclbebpeoaaijhoclanqepnaakqbjigdojlkpqgnlkcqef"}}
+{"translation":{"let":"hmbfeeiadfcmqnknipqfbgomgjkadakq", "rev" :"qkadakjgmogbfqpinknqmcfdaieefbmh"}}
+{"translation":{"let":"mo", "rev" :"om"}}
+{"translation":{"let":"fjablbnhokiedcdhp", "rev" :"phdcdeikohnblbajf"}}
+{"translation":{"let":"jhnmmhani", "rev" :"inahmmnhj"}}
+{"translation":{"let":"nmoamhfnbhdoiphd", "rev" :"dhpiodhbnfhmaomn"}}
+{"translation":{"let":"nfqhpofdegmkbpcolikpoqkopm", "rev" :"mpokqopkilocpbkmgedfophqfn"}}
+{"translation":{"let":"fkbqiehnnfqeqoom", "rev" :"mooqeqfnnheiqbkf"}}
+{"translation":{"let":"ilfaicflhofjjmqoeegk", "rev" :"kgeeoqmjjfohlfciafli"}}
+{"translation":{"let":"fhbdnaeicfopbddgqcjl", "rev" :"ljcqgddbpofcieandbhf"}}
+{"translation":{"let":"ablpjlgffoionbikekqidfnqljmoddkhea", "rev" :"aehkddomjlqnfdiqkekibnoioffgljplba"}}
+{"translation":{"let":"kon", "rev" :"nok"}}
+{"translation":{"let":"afdbqdkqhoahejpelgnkloccchpjlggceiiofmmnbcldcgqg", "rev" :"gqgcdlcbnmmfoiiecggljphcccolknglepjehaohqkdqbdfa"}}
+{"translation":{"let":"fgiigcgfqpokmdmophkdljh", "rev" :"hjldkhpomdmkopqfgcgiigf"}}
+{"translation":{"let":"fokedojicgiph", "rev" :"hpigcijodekof"}}
+{"translation":{"let":"icjpenbbcpifcgjhddcaolkan", "rev" :"nakloacddhjgcfipcbbnepjci"}}
+{"translation":{"let":"hlhkopqcnhabkibjqkocjm", "rev" :"mjcokqjbikbahncqpokhlh"}}
+{"translation":{"let":"clghiobfq", "rev" :"qfboihglc"}}
+{"translation":{"let":"dgncnppnqnmfoqdlifimcpajjmpjojeepgkgibhkanilfikeo", "rev" :"oekiflinakhbigkgpeejojpmjjapcmifildqofmnqnppncngd"}}
+{"translation":{"let":"iodhnekfnjbmeqpaca", "rev" :"acapqembjnfkenhdoi"}}
+{"translation":{"let":"khlolfpibfk", "rev" :"kfbipflolhk"}}
+{"translation":{"let":"hjlekcmdndinkp", "rev" :"pknidndmckeljh"}}
+{"translation":{"let":"cgemjdplnadoeq", "rev" :"qeodanlpdjmegc"}}
+{"translation":{"let":"gkiaegqnopelkqlhfopcdighac", "rev" :"cahgidcpofhlqkleponqgeaikg"}}
+{"translation":{"let":"hjcmfgmijhgbeigonjjqi", "rev" :"iqjjnogiebghjimgfmcjh"}}
+{"translation":{"let":"niponelgnqe", "rev" :"eqnglenopin"}}
+{"translation":{"let":"cfqnenkonajmdknilbcfglgnplbod", "rev" :"doblpnglgfcblinkdmjanoknenqfc"}}
+{"translation":{"let":"cqkfqaofhdcjniafpmbei", "rev" :"iebmpfainjcdhfoaqfkqc"}}
+{"translation":{"let":"knlnkmjkenlqp", "rev" :"pqlnekjmknlnk"}}
+{"translation":{"let":"cjhfkdoonfqdapepihjgpdgfpf", "rev" :"fpfgdpgjhipepadqfnoodkfhjc"}}
+{"translation":{"let":"ipqfhlbeebgqcdqbgbbaicjinkkdogochlnfpneclcnibddka", "rev" :"akddbinclcenpfnlhcogodkknijciabbgbqdcqgbeeblhfqpi"}}
+{"translation":{"let":"eclpohqf", "rev" :"fqhoplce"}}
+{"translation":{"let":"ocnikilfdbqhocmnhjgalifejelhqafgjaipkohmhdfjb", "rev" :"bjfdhmhokpiajgfaqhlejefilagjhnmcohqbdflikinco"}}
+{"translation":{"let":"eoeeibhkmhogkillhlbgbnhjfocpqhqqoicgajjlfoddogbmd", "rev" :"dmbgoddofljjagcioqqhqpcofjhnbgblhllikgohmkhbieeoe"}}
+{"translation":{"let":"cdpojifjbndfobmnejiljjmojonhqfdidiqekehflkqpc", "rev" :"cpqklfhekeqididfqhnojomjjlijenmbofdnbjfijopdc"}}
+{"translation":{"let":"ckcehejhfjp", "rev" :"pjfhjeheckc"}}
+{"translation":{"let":"lgplgdffqicglolnienmqajqkkcknmlejaqkd", "rev" :"dkqajelmnkckkqjaqmneinlolgciqffdglpgl"}}
+{"translation":{"let":"niolfclqnahgimdngflfaidcdbpibljhddh", "rev" :"hddhjlbipbdcdiaflfgndmighanqlcfloin"}}
+{"translation":{"let":"pgcfpnoknmmkhbdlbifhjebhlageigofidlcgjnmaaqlcq", "rev" :"qclqaamnjgcldifogiegalhbejhfibldbhkmmnkonpfcgp"}}
+{"translation":{"let":"hoknmkdaegfiqgcgdqknfbffgcgepablj", "rev" :"jlbapegcgffbfnkqdgcgqifgeadkmnkoh"}}
+{"translation":{"let":"dbgbdmjqfmgeqbidfalooplcdjjdgaaqlnjlecnloifkm", "rev" :"mkfiolnceljnlqaagdjjdclpoolafdibqegmfqjmdbgbd"}}
+{"translation":{"let":"dildqcnbihlkfjhcgqpajqbfblifqgncemidfnejqhnpmmmhgn", "rev" :"nghmmmpnhqjenfdimecngqfilbfbqjapqgchjfklhibncqdlid"}}
+{"translation":{"let":"qnekalgommcllnffapjhibgc", "rev" :"cgbihjpaffnllcmmoglakenq"}}
+{"translation":{"let":"ffcfklnpjifmlqjmbpgq", "rev" :"qgpbmjqlmfijpnlkfcff"}}
+{"translation":{"let":"fmbdaeqepgbpnnpfhnkg", "rev" :"gknhfpnnpbgpeqeadbmf"}}
+{"translation":{"let":"keondhhneimpehpfphkdfkpdoqgejbbqe", "rev" :"eqbbjegqodpkfdkhpfphepmienhhdnoek"}}
+{"translation":{"let":"qijcppbglonpjhfoebhnlfcdilnbnacjjalfdgmakgeofmq", "rev" :"qmfoegkamgdflajjcanbnlidcflnhbeofhjpnolgbppcjiq"}}
+{"translation":{"let":"mhffcenmodhppj", "rev" :"jpphdomnecffhm"}}
+{"translation":{"let":"djfkkhjibaaghqllligeoigb", "rev" :"bgioegilllqhgaabijhkkfjd"}}
+{"translation":{"let":"mmo", "rev" :"omm"}}
+{"translation":{"let":"omiqhheqcplkjcbqaqpqbaddnennjcghkmmidkfh", "rev" :"hfkdimmkhgcjnnenddabqpqaqbcjklpcqehhqimo"}}
+{"translation":{"let":"hcadoqjnmkilf", "rev" :"flikmnjqodach"}}
+{"translation":{"let":"jgmpeebjakplbbjamhminpfncplghmogccdkih", "rev" :"hikdccgomhglpcnfpnimhmajbblpkajbeepmgj"}}
+{"translation":{"let":"gelbbogqmcgmiklcilfcopmlionehhhdodkoakmjjdgkiqd", "rev" :"dqikgdjjmkaokdodhhhenoilmpocfliclkimgcmqgobbleg"}}
+{"translation":{"let":"dkqndjqcappkijgomohhcbacnklpfbndfgebidaegiglhlebf", "rev" :"fbelhlgigeadibegfdnbfplkncabchhomogjikppacqjdnqkd"}}
+{"translation":{"let":"hnlepbfcdqmfqcpiqobbeakmog", "rev" :"gomkaebboqipcqfmqdcfbpelnh"}}
+{"translation":{"let":"ednabgfmdjmigfj", "rev" :"jfgimjdmfgbande"}}
+{"translation":{"let":"qmfi", "rev" :"ifmq"}}
+{"translation":{"let":"jdkqgahqcgbfidnmlghodmmhafcbnpmklcpbai", "rev" :"iabpclkmpnbcfahmmdohglmndifbgcqhagqkdj"}}
+{"translation":{"let":"bfolnmibffeeheigljffhkdbn", "rev" :"nbdkhffjlgieheeffbimnlofb"}}
+{"translation":{"let":"ncliiah", "rev" :"haiilcn"}}
+{"translation":{"let":"bbdnkjcepnkifibpnhecgjnqjaq", "rev" :"qajqnjgcehnpbifiknpecjkndbb"}}
+{"translation":{"let":"jhnaabidkllfib", "rev" :"bifllkdibaanhj"}}
+{"translation":{"let":"mck", "rev" :"kcm"}}
+{"translation":{"let":"bhfhnljipjhdfokgaiofgidjeciallabq", "rev" :"qballaicejdigfoiagkofdhjpijlnhfhb"}}
+{"translation":{"let":"fjhkloiepbehdbeipefohlabfcfcqlceeomdnl", "rev" :"lndmoeeclqcfcfbalhofepiebdhebpeiolkhjf"}}
+{"translation":{"let":"icfehgglfbfbkkdnadkmp", "rev" :"pmkdandkkbfbflgghefci"}}
+{"translation":{"let":"ipjebbacgkpjecnqinjm", "rev" :"mjniqncejpkgcabbejpi"}}
+{"translation":{"let":"akapeiojceaaqnbomplnkobbjifmhadqcdgp", "rev" :"pgdcqdahmfijbboknlpmobnqaaecjoiepaka"}}
+{"translation":{"let":"neokiciombfbhboaedcq", "rev" :"qcdeaobhbfbmoicikoen"}}
+{"translation":{"let":"eiafgnflg", "rev" :"glfngfaie"}}
+{"translation":{"let":"egcnlf", "rev" :"flncge"}}
+{"translation":{"let":"kgmiqofjoldadammbkj", "rev" :"jkbmmadadlojfoqimgk"}}
+{"translation":{"let":"igldqkjkblic", "rev" :"cilbkjkqdlgi"}}
+{"translation":{"let":"gkgnheeigjgbfeajofjcomhpioddnldfkhaanpepljkglp", "rev" :"plgkjlpepnaahkfdlnddoiphmocjfojaefbgjgieehngkg"}}
+{"translation":{"let":"jiigeqelhlcfqfhobnjphkligohnidodekccao", "rev" :"oacckedodinhogilkhpjnbohfqfclhleqegiij"}}
+{"translation":{"let":"agplnbjeije", "rev" :"ejiejbnlpga"}}
+{"translation":{"let":"egehjhligd", "rev" :"dgilhjhege"}}
+{"translation":{"let":"dgfnfqmhdchnabnieglhllmkieqkoigqkjcfapd", "rev" :"dpafcjkqgiokqeikmllhlgeinbanhcdhmqfnfgd"}}
+{"translation":{"let":"jocgjhllacodkmnfljqfjbobg", "rev" :"gbobjfqjlfnmkdocallhjgcoj"}}
+{"translation":{"let":"laagaadjhmnqifqgomichdnqheihbmbdbkoiompbe", "rev" :"ebpmoiokbdbmbhiehqndhcimogqfiqnmhjdaagaal"}}
+{"translation":{"let":"dbfchgqihmpmeeanmoonhpgljhogphfpifqckceainlokjcdd", "rev" :"ddcjkolniaeckcqfipfhpgohjlgphnoomnaeempmhiqghcfbd"}}
+{"translation":{"let":"ndebjooabakojoqqeqhbbolbkphjqllhkpmjf", "rev" :"fjmpkhllqjhpkblobbhqeqqojokabaoojbedn"}}
+{"translation":{"let":"lqnlqppkccnbndjofpoaiajiikgcjjbbmlmkkhemmqfifaoddc", "rev" :"cddoafifqmmehkkmlmbbjjcgkiijaiaopfojdnbncckppqlnql"}}
+{"translation":{"let":"hpakdpdjgfnejkodfj", "rev" :"jfdokjenfgjdpdkaph"}}
+{"translation":{"let":"naeqqqondp", "rev" :"pdnoqqqean"}}
+{"translation":{"let":"ohdfeimqoejmphackodikhnajaoqfgmbebikfpininllndcd", "rev" :"dcdnllninipfkibebmgfqoajanhkidokcahpmjeoqmiefdho"}}
+{"translation":{"let":"qggmfdhp", "rev" :"phdfmggq"}}
+{"translation":{"let":"iohgjcigepmjmk", "rev" :"kmjmpegicjghoi"}}
+{"translation":{"let":"qpohdhafocfalploejgdpmqlmkdgaklknpiicqaaibjliik", "rev" :"kiiljbiaaqciipnklkagdkmlqmpdgjeolplafcofahdhopq"}}
+{"translation":{"let":"dnligfpmlead", "rev" :"daelmpfgilnd"}}
+{"translation":{"let":"feja", "rev" :"ajef"}}
+{"translation":{"let":"gnmieihi", "rev" :"ihieimng"}}
+{"translation":{"let":"qqbcpoengg", "rev" :"ggneopcbqq"}}
+{"translation":{"let":"ihbgaenomidfdagfcqo", "rev" :"oqcfgadfdimoneagbhi"}}
+{"translation":{"let":"ahfdemaejhmcaefeeiaqpcipbckldfnlodnmlokopqcjlhfj", "rev" :"jfhljcqpokolmndolnfdlkcbpicpqaieefeacmhjeamedfha"}}
+{"translation":{"let":"ggqjbnqedemjonpddahc", "rev" :"chaddpnojmedeqnbjqgg"}}
+{"translation":{"let":"akpjobfl", "rev" :"lfbojpka"}}
+{"translation":{"let":"qbm", "rev" :"mbq"}}

readme.md

@@ -1 +1,11 @@
 ## Readme
+
+- 'let' = word generated using random letters
+- 'rev' = reverse word generated
+
+
+### Goal
+- to train a model on this dataset so that it can generate reverse of words
+
+### Test DataSet
+- I am not using all the letters in my training set, so test set should be just words formed using remaining letters

requirements.txt

@@ -5,4 +5,5 @@ protobuf
 sacrebleu >= 1.4.12
 py7zr
 torch >= 1.3
-evaluate
+evaluate
+transformers

trainModel.py

@@ -0,0 +1,16 @@
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+
+tokenizer = T5Tokenizer.from_pretrained("t5-small")
+model = T5ForConditionalGeneration.from_pretrained("t5-small")
+
+
+max_source_length = 128
+max_target_length = 128
+
+input_ids = tokenizer("translate English to German: The house is wonderful.", return_tensors="pt").input_ids
+labels = tokenizer("Das Haus ist wunderbar.", return_tensors="pt").input_ids
+
+# the forward function automatically creates the correct decoder_input_ids
+loss = model(input_ids=input_ids, labels=labels).loss
+loss.item()
+print(loss.item())