Fixed issues with 'None', updated the ast algorithm functions so that they are consistent

21ea17c8 · s200545 · 87fc02e0 · 21ea17c8 · 21ea17c8 · 21ea17c8
Commit 21ea17c8 authored 2 years ago by s200545
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
+{
+  "python.analysis.extraPaths": [
+    "${workspaceFolder}\\__pypackages__\\3.10\\lib"
+  ],
+  "python.autoComplete.extraPaths": [
+    "${workspaceFolder}\\__pypackages__\\3.10\\lib"
+  ]
+}
\ No newline at end of file
--- a/README.md
+++ b/README.md
@@ -13,6 +13,8 @@ pdm run python -m formast <args>

 How to run: 
 ```
+2>&1 | tee log.txt 
+
 pdm run regit -vv --repo "C:\Users\boran\OneDrive\DTU\BSc Thesis\babyrepos\onlinebookstore" -o "C:\Users\boran\OneDrive\DTU\BSc Thesis\babyrepos\onlinebookstore_ast10" -p '**\*.java' -m mapping.csv -- formast --writeast --overwrite {}



--- a/csv_files/RxJava_csv/mapping_ast.csv
+++ b/csv_files/RxJava_csv/mapping_ast.csv
--- a/csv_files/RxJava_csv/mapping_comp_ast.csv
+++ b/csv_files/RxJava_csv/mapping_comp_ast.csv
--- a/csv_files/RxJava_csv/mapping_relative_ast.csv
+++ b/csv_files/RxJava_csv/mapping_relative_ast.csv
--- a/csv_files/RxJava_csv/mapping_sorted_comp_ast.csv
+++ b/csv_files/RxJava_csv/mapping_sorted_comp_ast.csv
--- a/csv_files/RxJava_csv/mapping_token.csv
+++ b/csv_files/RxJava_csv/mapping_token.csv
--- a/csv_files/bookstore_csv/mapping_ast.csv
+++ b/csv_files/bookstore_csv/mapping_ast.csv
--- a/csv_files/bookstore_csv/mapping_comp_ast.csv
+++ b/csv_files/bookstore_csv/mapping_comp_ast.csv
--- a/csv_files/bookstore_csv/mapping_relative_ast.csv
+++ b/csv_files/bookstore_csv/mapping_relative_ast.csv
--- a/csv_files/bookstore_csv/mapping_sorted_comp_ast.csv
+++ b/csv_files/bookstore_csv/mapping_sorted_comp_ast.csv
+from,to
+8a3399aec8341c0766a49826a023bb6b24562ec1,be9474a1320a2585581151aa211cf662595f593e
+a1e77a023e1449d6f1d6bb4fc321e537c1c91432,d605478e55d82a1590e016080e7810e95d3ff438
+7ee02cf2e1405fb376a7f0d9f10c53628ba18373,bf228ad9efb9eb8e0203b342352ade324fc1d241
+9f0a5c0e2efd594dfc93a7440f8965860b0fcfe0,abb99d108892d4a7e8baf858638f9896e3f46b94
+38443e531a68d35f9c3be2fcdbe749b3da232963,075870ec563a6ce2687cf8a3e8c801fc283826e7
+379714315d504754bc5ad0bc17ab6d224adf4ae6,26e0522eebfb91b664fb9afa7414a30b86c3fae6
+1b4b4c68c4ede676834ce62eb2d1f014fe3489a7,87a449925540f1c4c30cb89ac85d00fce3295bb7
+ac66a6b3dca0510e3e01eec02395966ef3e073fb,e153100f1816cadfd0bd98702bcb27746cd411e2
+1015838d6236681eee5655117636eda4d1156430,16b2e76e1688dc7cbda5fe7fd5d9f0f5d64d8543
+710dd9ab31c32670b0657d30b7b7a222b2d07593,bd43eebbb1facdb7ba7d81bdaab703cfbeb1fab3
+0c688cf10cba27a4b603fe51257ce80cc9cb8589,a8b0eeed26d6f2a6da3139fc5a77e214e21407ee
+05de44d435b198e643a2f84928c3ffa473e9fa91,a3d3fd1dc6e2d2a74f3f5a29b86a65fc2d6c1dd7
+b91b163ac356a423b7b308efca96fc793da4a582,15ee8eb141b14676aae7adaa62f49214a93029a3
+718ab80b0ff677a7a469175e2400a07ec44f62bd,b6563e19560a5559055f720d62c9ac1273aba522
+b09968e7dc17567757f00d5f9a848d30231d12c2,85295a4b31f3a606591c2266466b1b710ca37699
+78d978fb88e5987d44137c69647ab0c9f972a3d8,b9691188021b1856200f8eee71d540ba615e77df
+4d332bcfac027ef8c1ccf50a5f4a34b4ac653196,afcbb3aac5f40857b73219ca4f6c322cd54fde95
+0edb59e4c6051ca2b68f2d4f1321bb11bc849979,c9f1ea30495a45a05e4c9cc79eedb77c96d26b74
+9ec56e6505f581ebd5bc538e3904b4d60ec22d53,e14def006bcd84b7dbeb4ef830611e53dbe71eb4
+2dd23f668854bf818cfa872bbb0929e59d7dfe01,5df911de38ddab54c8da1e54f80b16f4159c8640
+153076191650ae84d447f9f841dcf2366e418b9e,11e41d38d599b62bd3a663dfe2426829b4da734d
+5c0550da855904ac4a19f4de3526ddc406f72adf,b51470d0c0d59ee88949b6b5b0ee7fc8c5222876
+93a8c9d509e797a17f5961e1b90e6b2e4a7a3581,027a951caa24b1888ef50137a254ce767e333b8b
+ecb6dff8ab4e723a66dd715e928bc6e19a10f053,91a682d8ef2bd823f539ac4b3ba41a72b04c6e4b
+33a359b4a37f32192fe60dcc86b4e8e2e1971769,ba778067e7c9c7e7f40b8e36753fac8ba4d04167
+54b775f07f20e37c233367d12dfe47abd689f858,c55292a998f1437bf91aedc0c16ead06c0b6b53d
+77c433b2c2d53fec73f1d3a8ab9d7355e2940013,4df7c484e1d6c45470a5e6661f5127e6d231f3f3
+12f04a23e5183129546416688525bf1e5b294f6e,d0813066d7f02d6bb7b90a3a0e556600f962651d
+3090be560d116c083eb4bd46908cebd9a9ca4094,71de55307e132828f53f0977854f1e0185d29e0b
+93ad50fb4da5924f851da683b6802df37c4ce450,c9dc3291644c87d8f21a0c0e72971f317084f40e
+35efe37fbe76c99591072c99ed4e34d7f6ef4333,af388b3ed962d1f4b30e546e390f63e208111c86
+ae814897adb9fa98038456fd3d6fd7a54f7191af,d73ff26292a6f80c44cdaa07d6bfc907b8839911
+35d289f123e2830c96afed55d6a1c1baeb92abce,7880d65a460238051eff8a574afe50d23f4b64ec
+8c685c737896297eb22614b0821cdb470431325a,5300ab69b732af5b27f5a02c7c866a516bbee838
+01db061505662ad6da55e360e76c791caca17d45,9d28a05372b50976bf873b775b266314e3204601
+8c0811fd7347f0b60bae8e24a061d7f98ab42f6e,4297318ec6ef0c0c2fd3a8cdef927956b13c0dba
+1ba3e2c094c65c29b1d10dc20f67312564e2aad3,813c11c6960aad54e731a5cbe56c8db9cbe3c7ba
+d0630f8451eb75836fdaee78cf0673f7b37da8cf,669b00f104ff7860b89c202e6cfea1ea6bbfaa1f
+c98cb07500b2a726ed76c3b63c99f6cecd1975f2,c8df5efd3d417adfbaecc2ecdce0f283250bc78a
+b633cf353f4ef73f7d707f46f5cf47c753979d2a,1288c569d4c186e268f37c1614091b2ad2438c4e
+fd644b23ea9a5df255c8b2ee3ceda56b7a010c7b,8c48f82e8f9526def5269fdc4890d86de9771c99
+898a6e3e0065903cc330de74fc66c50b64502496,89416cdd3b461fb9ad5c4f8789f750c9de637ff9
+779d44f710978bad686fb2adfd01bc4ec5ab6271,8f9fb0960354b0a9116ade9026af43c638cb6fe5
+5710819bda1b946fecea4a2e3d2e3f2b9e56d97f,36f61dc29b3b1f3387b8aea3d1f61e35c2c86fcf
+0fcec26daedee9565c818745389649af0b68b054,1eb4cd38cd7db0405e1cc9718f9d0b464e04de17
+8eb72782121ef4c7e8df907fad7ebaa10369404d,f17aab905ba25c2139dc94a9bcfe9f8dba846e9e
+d1e6703ef1566f7c87e63653ae7dd4661da3e340,47ec47c6a280bd3674a71445f576ebf466ab3d8f
+349d4c61f12081a92fb19276b76f7d73c7044c0c,290fc49458484c1d127aca8ee7e6c02b1eac0c72
+c8b1587a88c63853d085ce0e65502e2f4749a82b,6ab370614f6e5bde5d62c12e2d4c90b524057856
+100d50018e3631406c275da30ca933b2fba3720b,5baf7b513dd749324b4ae7914de1a3a49c5a9332
+8c8c76b2b5d698769b555898626c7032e6d739bf,bbe27b610c4466e37dd26e54ee93089361872df1
+c3daf8a4e419193e2945ccab129bedd290a5d8c1,8c6ee0238a28e5d940f5e3325e7b86043fafa848
+3fe64233d446cc4c7296d71249e149cf2eb22453,12732343e9f5658d1d07516f2df11f2299e98e2e
+9a0db509ff86bb0a844e5185f89e705e5b8e4b1f,accf4cd66a1c295b1f7611a356576b5f11fa8980
+3ba24c57878038e9167a35aed956f0a8af91f610,6a070f8a6f2e87f8306a47b07d06aa25530572ca
+34df97049825d1466fe1650ee67a3cfdd12b688a,41baee8a91df0f5662f5c1fe09f393075dba925d
+04a3eae65c8ec95df58b84b9daf56bef6c54d29c,a2eefc2633c4ca07d75768cd1dc90bc48b62aa2f
+2f2949108b4ba48a5237211bbd8a09402aa9a5d8,2212c51c6156e3a4d26baf1675c8489d8f0a1c77
+6d95bc1623fddeca4df402dd0884b7c4c9f73d2c,aead3bfbde71172beec7d1b960bf1b149bea3789
+f3aa58ac0bf41c031f004caa8deda4555e0b7a54,89f5fa077e3f6d6bbeb97586cf02d31c78b94b22
+22b14a1e08376458b32d280365d03f5e6ef8309b,a57af1cb8caddada50084f632ab3a4276256a9f8
+7b8494f9afb92c505d9dc173eb1bf6d22ccb5c44,9c8ea5bad036573d731ff152484727d76c5d80f1
+60fb31c444a157c3e69e53f599132c04c513a4c7,24de3ee7fe23c7ed709812f884030ec8861a2871
+13eb10f8a021cadb7df14c1847d66c84c14c6ae2,92eb52adfd2a064ad3563605ff32aaac1dbe5fe1
+1082a06a9056be0aa46a75bbf6469e1113d60822,21d07246d35f19ff93a9fe7039016b75c80aaf97
+4e6e56a4c94966255808798316312bb5c31d2e75,6d28717901299325fea94a38e3938a61b2d2b1ff
+cae53570875b772b7377f0645063fd1b99586994,b79be964f072c71119eb8883a0285d77049ff27a
+520a8c6f9266f4fc32e48f0114d60bbc0e9e7739,796b31e28d4906df62a54ad03ba46542b8b728a9
+bd90c600683f8299fd6faf3eed142f68986f0c95,049a714dc3c11ac45f7d4f441c286436dc22c94b
+d3c29c706cb794cac0a617f8d23fbfcee6b9635e,dab34217dec43e94c361ba908d6a4a02f436576a
+24ad22649e16a35558ff36e7a0fcf59ea374a62c,395882028964b20583d5761d811bde4a60147ec9
+c190a1a217cfd1c8b20fd019869fa00585befd8e,8a5450c5a45baba909bfb7824e6926da546a3219
+11cc00426d36b7650543e3fd2e1d43fb622f1cbd,ecc503ff153d91f855fe767e9d86c709087c7e95
+2c41475bb2a631c1e551ddfa7e4da2f5221ced77,c6146f30c5ccde93299099a253211ee53a8927e6
+9e10be9f4590136d71d7e18bf967d141fd21f08b,d125907906b76268bdbdab9eaca5e3ef72aeb70f
+45edd52796dc2025b46c9592678dc9be5f2b4178,6594e95f81aef83d009fd06982d0afd81b4160a3
+e99a9aa0f2a18ed7b7238d675bc8f0e6b69f5558,4d461075c5cce40a3287b87c2c278afcff447fec
+575c7fa688b3c82ea1b933f3a7ca29cce61f0f91,8c465f82f20cb62877651165d4ffad6703577c4d
+dfc52bf18596ab170eb0c81870394f0e8c7bbd15,f305c6548e31043216c7f19cc331237252e2816c
+ea1ee39e85e6d75aecede02aa0ad6e56c7106c50,e5364dd648f7141352896ccde98b38cf7c882102
+c073ae40dae8dd60359e6b87c0a326ba0f19dd37,3892739c124c3c8bb92033f2539cc84eaec8ee4c
+f058b754b4e913458a4eaee4b236c0652286ec7d,34bb3103b367aa99a2f5d2f9fc272a99acdce24f
+2041c637b6ddb0fb4cf021667b3d16e915822b1d,2f51c160b157a0b1595de21c4bf32efafffdaae0
+5ff2863f65b95115e5921282df17c40367887be8,5cfa108747eee806d06bfd50c1efdef0e414767d
+45f8d4e2b5d877543d0e327c4062d40ad23ba436,71cbb20e8c1bb9310fba7763c544c5108dc41c58
+3fc31b72bbc03aca2231cb6e3c72d2570fd7b686,c002a86f252d4bb95df864ef276dff77b2c15d0b
+823f43f84747679b7f805c6962502b650068af1f,97c73d8454e75a84c9e92ce76e888012dc65338c
+d292b12a491eb9e0903db8b3e43adfc500c5df12,7a6f763d4a4aaea076e0167ad099b51b1beb3619
+a58219f8c1eb9d54f04c11522a69179d6a474d4a,676c36d34e5b22bd9ad71c35db4ed94d62cf85b0
+65cabe5524ecc79cff553966fbdbffff22f7e4dc,6579d30a5101e1a9a6bf7d439cb7c3f2339dd7a5
+25a05a6470347a69c76159bc249f0985f1ea0a4f,4dbc2b1f6444344fb6c7e59fc95304662d296794
+c9cf5d1e841727e02fdb3d38fd5d01bf9f67d44c,2f484dec20f7b9cb854f33cf0327f117deec8dfc
+bac13a9702b03509cc938bf09898a2a8a84be0ec,c6a65720d3ce23832f37f79d1995ef9a02b9443f
+ac0a9f0b0891a80013707136c2bccce2d14d9320,cba52cebe9725575e2de39b7048852132ef37963
+2380c80d685f11cc67dc17ee56a521aafbd37de2,94eb94eb44a3a894030878f08d49b31f5976b7bf
+0c58e879a348111dbc6cbc7fbebf9ab89f6d0af5,c3714c4d3edb7c4e56fe5c67d919f620e9c9c42e
+cfa841e5260735ee8999d2465a5ce6038f652389,9ce670e8c313e23a71e152db2498cb8f009f2c68
+d41299022c6a0b31f62fbff65c394e20c0907b8b,a038d1c87eccdd66f354e9619e3b9678268c0ee1
+00a7b7898e8b47514fa665a4aefcda3afe18728c,174e35c1a8e68d556a3bfdc224a652464b7dca8e
+b11915dd7d770d26dfefa7fc50073173ef40e6fb,07f65ac0f453ac3abd421e79979535e81add36c7
+6327b22a49fe3ca8e4eaf8f814d5cf45721d24f1,5b25f12c7be591a7ca964ee334af2d97a1faee59
+4c438cccebe8795e46ef9ea51304ce6613cbbd1e,a1e1fda8aff73674d66b83cfef68697e4cc4231f
+5db9e98c2f875530bfda1334490b9fc37dec2a33,c88db4766d9920be08289b369ed4930c2089a9d1
+4e1032eac6f97d71f7d6da8f7e9693e636d2b551,1c9e76a2bbb723bd49ecb49a6ee708d753762b02
+bdc3504783897c9df75408c3b0b09426f46f3b26,ad0a58578207791db8688c7030e3145b542b0177
+6a9d338fb0fc66ef7c1dbb161e954a198b3fb815,187b07925fac0d1ceb4567d61cd94ddc920a9a3e
+5291effe50acb4cfb7f81c31c545c396fb2fb2ca,45712da00d6fc6408ca9e8084a66d0d41ee54d5b
+098fe44d8842b49954f7fca42210f16638df8a48,3ed2c1b11029576b151734dba3ccadcb3f824b52
+8741842e43506ca6dad2b74d13efd52859e30b77,0f4310ae17d21407395d5fae137107b094723477
+ba2371e84bae32d0184df25f991435b42feadbe9,00e3358661dc8b4a1d9c7ee6d8074a45cd671429
+202b58a87caf050d998df6172ec5fe5000e777b0,02a5079a9cd91a26a4f63ec9b9015a2409a22dab
+fcebedf83508739c2af857e928bd29e81bae229e,82d494e1c52a6678ba6c109c2b91d74b5d5cbd4f
+2823d37c0ad7b50d7ee7dec600453ef6f6de556e,bdfe40168582e19215a3aa0cf238b55b2838a04b
+f5acccceeb6b3fb6a940c86ee6ce41c1f2e4cbf9,12100ff19a6c22f8d81efd16ed01b4995d7c0f46
+3b452d39bfd0208565afdd884718c8ad3a08d77f,9c3abb2d19fa071c197d52accc25b00c8c39de00
+f6c7c422c333c0878b823b61edb4e48c0fc93544,cc6a36cf976af78af91ecde5922d7db32fced17a
+bfc449ad461fc77091500467bb493fdb06cf7df8,e5ffd82891c1b2845c44932e08d7c17d92bf313a
+3366a8fe5532ca9ca0b48744f5720d3b838cbcd6,83a6e32fa2a79182ec1d8179623bcd4a32e4fff2
+ea9551ce6629b92a957fc14125b5eb76a1e01236,cded5ac179031187545975eb192010318e76a1a0
+b2e4e0dec4a8fbb26fa369037e073d56198073f9,e8126f9132d8adede36b2a3bf793475f35d12a93
+2611ac5bc9560cb39692349f33ef4fff8bb2bf96,4a1ee7a6cec1966bedfb48f8f62afeef93cc5b8a
+abfb8eaecd32dd5f524bc52a24e50ec83c462128,f6aead9801bd8997cc0f62dd2b5f3aa14cea375e
+cb21184e74b76db5d13106475824235fdcdb9232,2bb5ba49e1760e90100cc3faf0c5aa9dce343753
+1bab13c706ab3486565c3ca74029ef391c56a7f1,34a260710aa7b2c51ebadb9a7476a24e765b391d
+d04a3df1bc11ada5d89eda7d038e5b48eb17b059,08c8565a78266bd16df5c3cf922294578d6a2b35
+5fd8daaf483c06ced9465341610df754438d57d1,ced16b34a427ce770f40b3b44d999a269094f4a2
+9e360f4846a029d438ebd9c0da067f01716f785e,976a52ff0ff442c6c6f30ba8df02a0188c57405e
+aa7bda81f5cda062e7749f5b23c8ab9238d3310e,73660ba964f2d46c629c6d54d79f3b3b72b63a70
+5c926f0aab24e4861c43500fe4af5343ece0e652,9a69ded5cf8ecaa8d8e1ed49bdf9baf86c55785c
+7fd8f8c0250a3b7087be3dc14f1440be52b78e54,2b98497d960908086aa22ffc78cc85a257482b6e
+b183f8a707676b6be9b809779281c8140baa54b5,25b43fe952823757d255ab01a915516453ca99c1
+361b6a355011765f07755a5d02d39aea17c7b13c,648b272ad295a0ad60d25109bbe8a31c98ead899
+0d8b6a1c503cb5ef127710c5be5de2debe5e7f85,f3db27326fb96ece5fc420cf4734df54c0ca9d5c
+cf063f4446cd67fc0a9f5deb9f126a24412a4d19,6d2c7e4d0819d02d2fcde7e324a6d7a29c61e59e
+23f31405b5bbc961be3359ddc9dc65e1900e910b,36b163affca302a1d76d1d53fce77e6db67f2086
+c0f8aaf8ad1b7d3e9c38489c2f0ba9f3bb9a3d94,3b1311db48018a7cc3c64a0ef1d4b61f6a920d7d
+6a7f7e5d427e0f08319a7e6fea561a4f1686eb7e,6585d11926aa6672d4c582c1f49d0bed75add052
+483b42b334a8099b155dd83bd0dd7f3456b5ffe4,9df0eee39bf65c7fe06e277c6dcef3ac015afaa1
+7b36f55008164f2a03bf9a1f5737619a9d381338,6cea287c976dd820c990095c900fc0d0320680c2
+d6de1c038259fff1f011b3fe26343b8cf34ffc7f,9849f87b475e6a58e7ae9624282fc19f32e3800a
+1730e2f8ea44d2b3a05eba2ee0f4a9cec07afd38,36d31ad60ae7df3d8a0c8f122a7f006aa2850a63
+4f086eff98a60697578c43475c1d3bbf1c4c9f4a,e35aaf5c42837f35bfee32b6310c6091a5117322
+17bafcff59f1aa4ec1ce20d732a878480cd64407,90661f23f284a2cd1ca33c104953db0e37272dae
+5708372ad139701a297b6c3ab2a2d9d72c0ef528,fbbceb42f1ff2308953f4c8592bd3a86ef1cfe65
+d7583a8473e99ac99ffc9c75d4b01611cffabe23,abddf32f2dd7ad40e5af4f74b0b5674edea71e1e
+fad4c98b7993d8604bbc5dae9a0809c974c0f9e8,30f4804d9b299786decdef5b67c0bf5e5d1a1afc
+33b4178ff678632ebee886fb4573a0c3f4744921,7d20884ea95602d28e1262d0df18238ced54ac72
+636cedf5a969371e87891424b85cdc395e8c02f1,d149e84c8e4424a5a94f26ffcaa396c3db30ee90
+cf7305d378fbc5f773a3ddbf05991693fde6ffd9,5700152cc35e34427cd2781e013b2d4db6215a4d
+425a890eeda38f0fe65161fa9e1480c53440de06,0d2e8b61ad726f737ea24a1a2c4a954e05d879a6
+895efdec72daf938de4f5f0a1be36d0c2354e94a,626494f31a96e404917500ca14f33a4703cc614c
+67af06a58bd44ee4b208b8ac7b14e184661bb207,8b08f40001b12bba91f5b4211ff27b58d750a50f
+cbd17562080602e48c79278ae28f7cb120ddd6a9,b126622ba4c4f4935b48f5bb72761b738ac846e9
+99b3c617cc06c7b0f43d66f94ab7f02173498af9,111b2aabdc9aec8db65c4bede004d7d0b35c021b
+e1f593987acdc912c1e28d83dac919b8bbb109b0,2a279d2becc8770a12469644bdb342c3edb850af
+5f96561f488ea53dacda12e2396b5baf467d580a,ced2c757c2ff68c2a1529d9a2527db3d2e7ac26d
+611d2126ac238932ee96a251e888243f53730ab7,64e8ca857602709f690ad4f71f93c5e9fa8b5f7e
+1225f9cdd9e1c5b82c0b488503851e61887e6b08,c885598c371347e9adc28cfc1e8fd0d1d202dec2
+ee9b3a66254a9f8030fba862151370071a830e1a,69ee802a15e27a68d5990959af893012bb6cab4f
+716732f92c142e5e064eb91007bff972037c36bb,8176c085d1ce9abbd6a5d78bc66743892fb6d127
+a2c5dfb20b75962a3f8520eb63f223b79b82c0bc,c3aae3c8f804b84c6035ba0cf5ff5987da7916d6
+e979b65d2641439f6533ed934715e6aefd48dda7,b6c45497b748013284496aa6088c662af57640a2
+fad3501514c004535e0f18d9575c2a1af9f724be,31cc2dabc88e60cdd3d91fb39eb309868caefe10
+5085b156acd0f5e6080843d7529fab27aa07844e,967b1d8c0f102796f87f6f823a859755922ef175
+d6a3ef6a50d7b4fe2bc258c90b7258ae95786a4f,0a7cc423f6bb431a8dfba67b4697494e65c8631b
+1a7c915114658366031cf681156842835628b7e4,e30ffb7c4321e79ba4ba306133d9ae7752c1dcb1
+81af04554880f72813db468c99ea9ea747ba4ca1,b316aee02f811030a8d55d07965852a534a34c07
+629aaf4c48233ad012a42b9d931a34936f1df756,39e08b1b49b76e7485bf36b613d224a9e50692b7
+5a301fa9c7ee3240ba020c608f02df1274c514db,8d421befb242783e3497c55e66423b90e2465e33
+9c5aaece593946f75ff8ac1173378c0283d3fbea,15db7ad2037e34206c683452d56687281892df26
+aab31d5d465922612088e2ac9f64c1232c42cd35,7d4a51884b20b19db2b0a948fd8d866514aade49
+6f026f7624fd69f65a80f50574a7405e5df22231,df3add7a3c6b3c179e810de603c89f26c8b3966c
+4d2160c9a46df1e1686001a5b78649587eaa6a4f,1b45e92d872c922515ee7fc67b126c38190960bc
+e376b0a5c631b2747896fdc9d37c1b05ccbf45aa,9bf7f85243c0e7737fab0a815cda5b0a528a116b
+3e24449df1f320d220c0d20dedada8fcb69c4fcc,ef201203e25fb66bacf137541b4198fd8e0d8d25
+a63aebd42925d46aa4fdd469bca7cde7689415f2,4c3325bb156026eaec09ab7aca172a9fb5b5fad1
+f4da33d4e49d67b15ca26c920d7bd3189087c6f7,c89d16a1ff57284d3fbd946177bfa3c2fb6dfa27
+25ea00539c2148b2e7521b18c900288f7ca97884,81902765dbac1201f810815a469e4d62b29ce39e
+5d0fb783450d71fb8905da346b20fc0cbbde2c5d,5f0c9f719d053dae2a99bc67265981755b6120f3
+489f4f07b41eb891b42385f651c9b869452245fa,01e008ca6540f2799c1bb9b0e68d9b2e9c1ca848
+4a307fabea604be9cb5bf92a5c65c29e9f0028f4,30ebe2f5bd48d89f098a1081c3ef0cf7df683cee
+2c09462e3198843acecb1be04007ca7ad256ef64,5f41dc3bdd0813df510ed3c97ec9c89ffeecb23e
+4e03872a5b14589b632cec525165514d2b1c05a5,7697e13e0a48d86873b87a733976112cff7ad35e
+e46ad824f62d1b400c69e05a15d70518fdaadd51,1a7c3380aecbeae17bbf05d4174a7895050e6961
+408ef28e190e86375d96bae5277a5fb04b7522bf,1c467df61fe89595505a4819d5f6dbceec895232
+ba5e7a6d5df4ca95e2306c062947e5df3b1026cd,b1ad06cc212dcd5fb3673f76a50e02453b81a497
+2e7168a0de7650eb3dfb57b1dcf0bb1c637c4aa6,d36f8c377f122a941c9500737df07d33c2b3b674
+668f6fb87f546fdb6d61b19913868ae6cb37c588,64c8080a31be0018e904616b8c15a7f1fa593299
+ad8f2fa5c14b9c2eda1bab19fc0eb7f33a9ece44,6788847fb543236ecdb4d8c098a787e468eb0863
+c3a17ad68b4b55e5feb6f194de6754535b1bac69,e7840bf4778197492e23ffe17d5932249a98423b
+59e4f86a4d265dd166aaabb5fd3277b07668881f,a9a510eeca56bd89a5c442fbe0cf6071e4680b39
+04bfbd44fc64324dceab22b069c90bb723ea9c49,48034ae68c522f59a658d8b3b4516dd0416de1e1
+7f69dcee1135798e8b06e2fbf8539f7f04adb6d6,db8f03e3a778f39621ed228c4fa8a98024401226
+011fad3dc5cd82ee511b196130f5ea7b54cc6ea0,e2437a3504db9b0b561522cd525d3339f78ce21a
+d604f45e28e56be479a20c2e0a52fbe3a757e2c5,7560ae593bd7d21384b035e2f8b15da9681bc181
--- a/csv_files/bookstore_csv/output_ast.csv
+++ b/csv_files/bookstore_csv/output_ast.csv
--- a/csv_files/bookstore_csv/output_comp_ast.csv
+++ b/csv_files/bookstore_csv/output_comp_ast.csv
--- a/csv_files/bookstore_csv/output_relative_ast.csv
+++ b/csv_files/bookstore_csv/output_relative_ast.csv
--- a/csv_files/bookstore_csv/output_sort_comp_ast.csv
+++ b/csv_files/bookstore_csv/output_sort_comp_ast.csv
--- a/eval/Testing area.ipynb
+++ b/eval/Testing area.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "363be74b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<tree_sitter.Tree at 0x203c968bef0>"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from collections import deque\n",
+    "from tree_sitter import Language, Parser\n",
+    "import os\n",
+    "from formast.__main__ import JAVA_LANGUAGE\n",
+    "\n",
+    "parser = Parser()\n",
+    "parser.set_language(JAVA_LANGUAGE)\n",
+    "\n",
+    "with open(\"..\\java_files\\simple.java\", \"rb\") as f:\n",
+    "    code = f.read()\n",
+    "tree = parser.parse(code)\n",
+    "tree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4997940c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "program [0, 0] - [0, 16]\n",
+      "  class_declaration [0, 0] - [0, 16]\n",
+      "    modifiers [0, 0] - [0, 6]\n",
+      "      public [0, 0] - [0, 6]\n",
+      "    class [0, 7] - [0, 12]\n",
+      "    identifier [0, 13] - [0, 14]\n",
+      "    class_body [0, 14] - [0, 16]\n",
+      "      { [0, 14] - [0, 15]\n",
+      "      } [0, 15] - [0, 16]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from tree_sitter import Node\n",
+    "\n",
+    "def print_node(node: Node, indent: str = \"\"):\n",
+    "    print(f\"{indent}{node.type} [{node.start_point[0]}, {node.start_point[1]}] - [{node.end_point[0]}, {node.end_point[1]}]\")\n",
+    "    for child in node.children:\n",
+    "        print_node(child, indent + \"  \")\n",
+    "\n",
+    "print_node(tree.root_node)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "c354d941",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "B public \n",
+      "B modifiers 0\n",
+      "B class \n",
+      "N A\n",
+      "B identifier 3\n",
+      "B class_body \n",
+      "B class_declaration 1 2 4 5\n",
+      "B program 6\n"
+     ]
+    }
+   ],
+   "source": [
+    "def process_tree(tree, code):\n",
+    "    lines = []\n",
+    "    def process_node(node):\n",
+    "        if node.type == 'identifier':\n",
+    "            #it was decode before\n",
+    "            lines.append(f'N {code[node.start_byte:node.end_byte].decode(\"utf-8\")}')\n",
+    "            lines.append(f'B identifier {len(lines) - 1}')\n",
+    "            return len(lines) - 1\n",
+    "        elif node.type == 'class_body':\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                if child.type not in ['{', '}']:\n",
+    "                    child_index = process_node(child)\n",
+    "                    if child_index is not None:\n",
+    "                        children.append(child_index)\n",
+    "            lines.append(f'B {node.type} {\" \".join(map(str, children))}')\n",
+    "            return len(lines) - 1\n",
+    "        else:\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                child_index = process_node(child)\n",
+    "                if child_index is not None:\n",
+    "                    children.append(child_index)\n",
+    "            lines.append(f'B {node.type} {\" \".join(map(str, children))}')\n",
+    "            return len(lines) - 1\n",
+    "    process_node(tree.root_node)\n",
+    "    return '\\n'.join(lines)\n",
+    "\n",
+    "print(process_tree(tree,code))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "c28c55c3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "L public\n",
+      "B modifiers 0\n",
+      "L class\n",
+      "L A\n",
+      "L {\n",
+      "L }\n",
+      "B class_body 4 5\n",
+      "B class_declaration 1 2 3 6\n",
+      "B program 7\n"
+     ]
+    }
+   ],
+   "source": [
+    "def process_tree_ast(tree):\n",
+    "    lines = []\n",
+    "\n",
+    "    def process_node(node):\n",
+    "        line = \"\"\n",
+    "\n",
+    "        if node.children:\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                child_result = process_node(child)\n",
+    "                if child_result is not None:\n",
+    "                    children.append(child_result)\n",
+    "            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))\n",
+    "        else:\n",
+    "            text = node.text.decode('utf-8')\n",
+    "            line = 'L {}'.format(text)\n",
+    "\n",
+    "        lines.append('{}'.format(line))\n",
+    "\n",
+    "        return len(lines) - 1 \n",
+    "\n",
+    "    process_node(tree.root_node)\n",
+    "    return '\\n'.join(lines) \n",
+    "\n",
+    "print(process_tree_ast(tree))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "45bf0be8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "L public\n",
+      "B modifiers -1\n",
+      "L class\n",
+      "L A\n",
+      "L {\n",
+      "L }\n",
+      "B class_body -2 -1\n",
+      "B class_declaration -6 -5 -4 -1\n",
+      "B program -1\n"
+     ]
+    }
+   ],
+   "source": [
+    "def process_tree_ast_rel(tree):\n",
+    "    if tree is None:\n",
+    "        raise ValueError(\"The tree object must not be None\")\n",
+    "\n",
+    "    lines = []\n",
+    "\n",
+    "    def process_node(node):\n",
+    "        if node is None:\n",
+    "            raise ValueError(\"The tree object does not have the expected structure\")\n",
+    "\n",
+    "        line = \"\"\n",
+    "\n",
+    "        if node.children:\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                child_result = process_node(child)\n",
+    "                if child_result is not None:\n",
+    "                    children.append(child_result)\n",
+    "            line = 'B {} {}'.format(node.type, ' '.join(map(lambda x: str(x - len(lines)), children)))\n",
+    "        else:\n",
+    "            try:\n",
+    "                text = node.text.decode('utf-8')\n",
+    "            except UnicodeDecodeError:\n",
+    "                raise ValueError(\"The text of the leaf nodes must be encoded using utf-8\")\n",
+    "            line = 'L {}'.format(text)\n",
+    "\n",
+    "        lines.append('{}'.format(line))\n",
+    "\n",
+    "        return len(lines) - 1\n",
+    "\n",
+    "    process_node(tree.root_node)\n",
+    "    return '\\n'.join(lines)\n",
+    "\n",
+    "print(process_tree_ast_rel(tree))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "4b6483eb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "_6xEEtLT7vg L public\n",
+      "iLRoi73V-RI B modifiers _6xEEtLT7vg\n",
+      "2usUse1FW98 L class\n",
+      "Jf7gA03GbKE L A\n",
+      "f1dtMFaAKXw L {\n",
+      "QRb-VDvN7n0 L }\n",
+      "_ETJtzenYkI B class_body f1dtMFaAKXw QRb-VDvN7n0\n",
+      "FwqmimPqmxY B class_declaration iLRoi73V-RI 2usUse1FW98 Jf7gA03GbKE _ETJtzenYkI\n",
+      "M8gnXQfEJJs B program FwqmimPqmxY\n"
+     ]
+    }
+   ],
+   "source": [
+    "import base64\n",
+    "import hashlib\n",
+    "\n",
+    "def process_tree_comp(tree):\n",
+    "    lookup = {}\n",
+    "    lines = []\n",
+    "\n",
+    "    def process_node(node):\n",
+    "        line = \"\"\n",
+    "\n",
+    "        if node.children:\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                child_result = process_node(child)\n",
+    "                if child_result is not None:\n",
+    "                    children.append(child_result)\n",
+    "            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))\n",
+    "        else:\n",
+    "            text = node.text.decode('utf-8')\n",
+    "            line = 'L {}'.format(text)\n",
+    "\n",
+    "        idx = lookup.get(line)\n",
+    "        if idx is not None:\n",
+    "            return idx\n",
+    "\n",
+    "        hash_object = hashlib.sha256(line.encode('utf-8'))\n",
+    "        hash_value = int.from_bytes(hash_object.digest()[:8], byteorder='big', signed=True)\n",
+    "        idx = base64.urlsafe_b64encode(hash_value.to_bytes(8, byteorder='big', signed=True)).rstrip(b'=').decode('ascii')\n",
+    "        lines.append('{} {}'.format(idx, line))\n",
+    "        lookup[line] = idx\n",
+    "\n",
+    "        return idx\n",
+    "\n",
+    "    process_node(tree.root_node)\n",
+    "    return '\\n'.join(lines)  \n",
+    "\n",
+    "print(process_tree_comp(tree))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "17eb22cf",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2usUse1FW98 L class\n",
+      "FwqmimPqmxY B class_declaration iLRoi73V-RI 2usUse1FW98 Jf7gA03GbKE _ETJtzenYkI\n",
+      "Jf7gA03GbKE L A\n",
+      "M8gnXQfEJJs B program FwqmimPqmxY\n",
+      "QRb-VDvN7n0 L }\n",
+      "_6xEEtLT7vg L public\n",
+      "_ETJtzenYkI B class_body f1dtMFaAKXw QRb-VDvN7n0\n",
+      "f1dtMFaAKXw L {\n",
+      "iLRoi73V-RI B modifiers _6xEEtLT7vg\n"
+     ]
+    }
+   ],
+   "source": [
+    "import base64\n",
+    "import hashlib\n",
+    "\n",
+    "def process_tree_comp_sorted(tree):\n",
+    "    if tree is None:\n",
+    "        raise ValueError(\"The tree object must not be None\")\n",
+    "\n",
+    "    lookup = {}\n",
+    "    lines = []\n",
+    "\n",
+    "    def process_node_sorted(node):\n",
+    "        if node is None:\n",
+    "            raise ValueError(\"The tree object does not have the expected structure\")\n",
+    "\n",
+    "        line = \"\"\n",
+    "\n",
+    "        if node.children:\n",
+    "            children = []\n",
+    "            for child in node.children:\n",
+    "                child_result = process_node_sorted(child)\n",
+    "                if child_result is not None:\n",
+    "                    children.append(child_result)\n",
+    "            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))\n",
+    "        else:\n",
+    "            try:\n",
+    "                text = node.text.decode('utf-8')\n",
+    "            except UnicodeDecodeError:\n",
+    "                raise ValueError(\"The text of the leaf nodes must be encoded using utf-8\")\n",
+    "            line = 'L {}'.format(text)\n",
+    "\n",
+    "        idx = lookup.get(line)\n",
+    "        if idx is not None:\n",
+    "            return idx\n",
+    "\n",
+    "        hash_object = hashlib.sha256(line.encode('utf-8'))\n",
+    "        hash_value = int.from_bytes(hash_object.digest()[:8], byteorder='big', signed=True)\n",
+    "        idx = base64.urlsafe_b64encode(hash_value.to_bytes(8, byteorder='big', signed=True)).rstrip(b'=').decode('ascii')\n",
+    "        lines.append('{} {}'.format(idx, line))\n",
+    "        lookup[line] = idx\n",
+    "\n",
+    "        return idx\n",
+    "\n",
+    "    process_node_sorted(tree.root_node)\n",
+    "    sorted_lines = sorted(lines, key=lambda x: x.split()[0])\n",
+    "    return '\\n'.join(sorted_lines)\n",
+    "\n",
+    "print(process_tree_comp_sorted(tree))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+%% Cell type:code id:363be74b tags:
+
+``` python
+from collections import deque
+from tree_sitter import Language, Parser
+import os
+from formast.__main__ import JAVA_LANGUAGE
+
+parser = Parser()
+parser.set_language(JAVA_LANGUAGE)
+
+with open("..\java_files\simple.java", "rb") as f:
+    code = f.read()
+tree = parser.parse(code)
+tree
+```
+
+%% Output
+
+    <tree_sitter.Tree at 0x203c968bef0>
+
+%% Cell type:code id:4997940c tags:
+
+``` python
+from tree_sitter import Node
+
+def print_node(node: Node, indent: str = ""):
+    print(f"{indent}{node.type} [{node.start_point[0]}, {node.start_point[1]}] - [{node.end_point[0]}, {node.end_point[1]}]")
+    for child in node.children:
+        print_node(child, indent + "  ")
+
+print_node(tree.root_node)
+```
+
+%% Output
+
+    program [0, 0] - [0, 16]
+      class_declaration [0, 0] - [0, 16]
+        modifiers [0, 0] - [0, 6]
+          public [0, 0] - [0, 6]
+        class [0, 7] - [0, 12]
+        identifier [0, 13] - [0, 14]
+        class_body [0, 14] - [0, 16]
+          { [0, 14] - [0, 15]
+          } [0, 15] - [0, 16]
+
+%% Cell type:code id:c354d941 tags:
+
+``` python
+def process_tree(tree, code):
+    lines = []
+    def process_node(node):
+        if node.type == 'identifier':
+            #it was decode before
+            lines.append(f'N {code[node.start_byte:node.end_byte].decode("utf-8")}')
+            lines.append(f'B identifier {len(lines) - 1}')
+            return len(lines) - 1
+        elif node.type == 'class_body':
+            children = []
+            for child in node.children:
+                if child.type not in ['{', '}']:
+                    child_index = process_node(child)
+                    if child_index is not None:
+                        children.append(child_index)
+            lines.append(f'B {node.type} {" ".join(map(str, children))}')
+            return len(lines) - 1
+        else:
+            children = []
+            for child in node.children:
+                child_index = process_node(child)
+                if child_index is not None:
+                    children.append(child_index)
+            lines.append(f'B {node.type} {" ".join(map(str, children))}')
+            return len(lines) - 1
+    process_node(tree.root_node)
+    return '\n'.join(lines)
+
+print(process_tree(tree,code))
+```
+
+%% Output
+
+    B public
+    B modifiers 0
+    B class
+    N A
+    B identifier 3
+    B class_body
+    B class_declaration 1 2 4 5
+    B program 6
+
+%% Cell type:code id:c28c55c3 tags:
+
+``` python
+def process_tree_ast(tree):
+    lines = []
+
+    def process_node(node):
+        line = ""
+
+        if node.children:
+            children = []
+            for child in node.children:
+                child_result = process_node(child)
+                if child_result is not None:
+                    children.append(child_result)
+            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))
+        else:
+            text = node.text.decode('utf-8')
+            line = 'L {}'.format(text)
+
+        lines.append('{}'.format(line))
+
+        return len(lines) - 1
+
+    process_node(tree.root_node)
+    return '\n'.join(lines)
+
+print(process_tree_ast(tree))
+```
+
+%% Output
+
+    L public
+    B modifiers 0
+    L class
+    L A
+    L {
+    L }
+    B class_body 4 5
+    B class_declaration 1 2 3 6
+    B program 7
+
+%% Cell type:code id:45bf0be8 tags:
+
+``` python
+def process_tree_ast_rel(tree):
+    if tree is None:
+        raise ValueError("The tree object must not be None")
+
+    lines = []
+
+    def process_node(node):
+        if node is None:
+            raise ValueError("The tree object does not have the expected structure")
+
+        line = ""
+
+        if node.children:
+            children = []
+            for child in node.children:
+                child_result = process_node(child)
+                if child_result is not None:
+                    children.append(child_result)
+            line = 'B {} {}'.format(node.type, ' '.join(map(lambda x: str(x - len(lines)), children)))
+        else:
+            try:
+                text = node.text.decode('utf-8')
+            except UnicodeDecodeError:
+                raise ValueError("The text of the leaf nodes must be encoded using utf-8")
+            line = 'L {}'.format(text)
+
+        lines.append('{}'.format(line))
+
+        return len(lines) - 1
+
+    process_node(tree.root_node)
+    return '\n'.join(lines)
+
+print(process_tree_ast_rel(tree))
+```
+
+%% Output
+
+    L public
+    B modifiers -1
+    L class
+    L A
+    L {
+    L }
+    B class_body -2 -1
+    B class_declaration -6 -5 -4 -1
+    B program -1
+
+%% Cell type:code id:4b6483eb tags:
+
+``` python
+import base64
+import hashlib
+
+def process_tree_comp(tree):
+    lookup = {}
+    lines = []
+
+    def process_node(node):
+        line = ""
+
+        if node.children:
+            children = []
+            for child in node.children:
+                child_result = process_node(child)
+                if child_result is not None:
+                    children.append(child_result)
+            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))
+        else:
+            text = node.text.decode('utf-8')
+            line = 'L {}'.format(text)
+
+        idx = lookup.get(line)
+        if idx is not None:
+            return idx
+
+        hash_object = hashlib.sha256(line.encode('utf-8'))
+        hash_value = int.from_bytes(hash_object.digest()[:8], byteorder='big', signed=True)
+        idx = base64.urlsafe_b64encode(hash_value.to_bytes(8, byteorder='big', signed=True)).rstrip(b'=').decode('ascii')
+        lines.append('{} {}'.format(idx, line))
+        lookup[line] = idx
+
+        return idx
+
+    process_node(tree.root_node)
+    return '\n'.join(lines)
+
+print(process_tree_comp(tree))
+```
+
+%% Output
+
+    _6xEEtLT7vg L public
+    iLRoi73V-RI B modifiers _6xEEtLT7vg
+    2usUse1FW98 L class
+    Jf7gA03GbKE L A
+    f1dtMFaAKXw L {
+    QRb-VDvN7n0 L }
+    _ETJtzenYkI B class_body f1dtMFaAKXw QRb-VDvN7n0
+    FwqmimPqmxY B class_declaration iLRoi73V-RI 2usUse1FW98 Jf7gA03GbKE _ETJtzenYkI
+    M8gnXQfEJJs B program FwqmimPqmxY
+
+%% Cell type:code id:17eb22cf tags:
+
+``` python
+import base64
+import hashlib
+
+def process_tree_comp_sorted(tree):
+    if tree is None:
+        raise ValueError("The tree object must not be None")
+
+    lookup = {}
+    lines = []
+
+    def process_node_sorted(node):
+        if node is None:
+            raise ValueError("The tree object does not have the expected structure")
+
+        line = ""
+
+        if node.children:
+            children = []
+            for child in node.children:
+                child_result = process_node_sorted(child)
+                if child_result is not None:
+                    children.append(child_result)
+            line = 'B {} {}'.format(node.type, ' '.join(map(str, children)))
+        else:
+            try:
+                text = node.text.decode('utf-8')
+            except UnicodeDecodeError:
+                raise ValueError("The text of the leaf nodes must be encoded using utf-8")
+            line = 'L {}'.format(text)
+
+        idx = lookup.get(line)
+        if idx is not None:
+            return idx
+
+        hash_object = hashlib.sha256(line.encode('utf-8'))
+        hash_value = int.from_bytes(hash_object.digest()[:8], byteorder='big', signed=True)
+        idx = base64.urlsafe_b64encode(hash_value.to_bytes(8, byteorder='big', signed=True)).rstrip(b'=').decode('ascii')
+        lines.append('{} {}'.format(idx, line))
+        lookup[line] = idx
+
+        return idx
+
+    process_node_sorted(tree.root_node)
+    sorted_lines = sorted(lines, key=lambda x: x.split()[0])
+    return '\n'.join(sorted_lines)
+
+print(process_tree_comp_sorted(tree))
+```
+
+%% Output
+
+    2usUse1FW98 L class
+    FwqmimPqmxY B class_declaration iLRoi73V-RI 2usUse1FW98 Jf7gA03GbKE _ETJtzenYkI
+    Jf7gA03GbKE L A
+    M8gnXQfEJJs B program FwqmimPqmxY
+    QRb-VDvN7n0 L }
+    _6xEEtLT7vg L public
+    _ETJtzenYkI B class_body f1dtMFaAKXw QRb-VDvN7n0
+    f1dtMFaAKXw L {
+    iLRoi73V-RI B modifiers _6xEEtLT7vg
--- a/eval/evaluation.ipynb
+++ b/eval/evaluation.ipynb
--- a/eval/evaluation_onlinebookstore.ipynb
+++ b/eval/evaluation_onlinebookstore.ipynb
--- a/eval/evalutation_rxjava.ipynb
+++ b/eval/evalutation_rxjava.ipynb
--- a/eval/simple.ast
+++ b/eval/simple.ast