Add download_logs and scripts

.gitattributes

@@ -16,3 +16,4 @@
 *.pth filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 flax_model.msgpack filter=lfs diff=lfs merge=lfs -text
+*.log filter=lfs diff=lfs merge=lfs -text

data/download_logs/cc12m_2m_download.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e00288c04e8d226862a37c1ca4d4953a40770d30f7f7f696aafcba2ed57212d
+size 145262942

data/download_logs/cc12m_download.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c25de5a6a1bffc69309fdf30d53a27978bb0de8b49a80429b14f880c6470495b
+size 262929928

data/download_logs/cc3m_download.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f5aeaaf47c1370a5da33fc6a97303a1ef5d020670e06fbc9e8474b41a5eb3ba
+size 126513213

data/download_logs/wit_download.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:361c1a8fd3704ec101be204690949ff20b02eac54c39c9fb4ed934d0497ff6ba
+size 233568

data/scripts/cc12m.py

@@ -0,0 +1,49 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing cc12m dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url"]]
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url in df.itertuples():
+    index+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+
+with open(output_file, "w") as f:
+    f.write("\n".join(lines))
+
+logging.info(f"Processing cc12m dataset done. {len(lines)} images processed.")
+

data/scripts/cc12m_disk1.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing cc12m dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url"]]
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url in df.itertuples():
+    index+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+train_lines = lines[:-150_001]
+valid_lines = lines[-150_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing cc12m dataset done. {len(lines)} images processed.")
+

data/scripts/cc12m_disk2.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing cc12m dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url"]]
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url in df.itertuples():
+    index+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+train_lines = lines[:-500_001]
+valid_lines = lines[-500_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing cc12m dataset done. {len(lines)} images processed.")
+

data/scripts/cc3m.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing cc3m dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url"]]
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url in df.itertuples():
+    index+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+train_lines = lines[:-300_001]
+valid_lines = lines[-300_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing cc3m dataset done. {len(lines)} images processed.")
+

data/scripts/cc3m_modified.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing cc3m dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url", "index_row"]]
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url, index_row in df.itertuples():
+    index_row+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index_row:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+train_lines = lines[:-300_001]
+valid_lines = lines[-300_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing cc3m dataset done. {len(lines)} images processed.")
+

data/scripts/cc_propn.py

@@ -0,0 +1,72 @@
+import sys
+import os
+from datetime import datetime
+import pandas as pd
+import contexttimer
+from urllib.request import urlopen
+import requests
+from PIL import Image
+import torch
+from torchvision.transforms import functional as TF
+from multiprocessing import Pool
+from tqdm import tqdm
+import logging
+import sys
+import numpy as np
+
+
+
+from nltk.tag import CRFTagger
+ct = CRFTagger()
+ct.set_model_file('all_indo_man_tag_corpus_model.crf.tagger')
+
+headers = {
+    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
+    "X-Forwarded-For": "64.18.15.200",
+}
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+
+'''if len(sys.argv) != 3:
+    print("Provide .tsv file name & output directory. e.g. python downloader.py Train-GCC-training.tsv training")
+    exit(1)'''
+
+# Load data
+print(f'Starting to load at {datetime.now().isoformat(timespec="minutes")}')
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(sys.argv[1], delimiter='\t')
+    df = df[["caption", "url"]]
+
+def drop_no(text):
+    try:
+        if len(text)==0:
+            return True
+        elif len(text) > 96:
+            return True
+        text = text.split()
+        result = ct.tag_sents([text])
+        nnp_cnt = 0
+        total = len(result[0])
+
+        for x in result[0]:
+            if x[1] == "NNP":
+                nnp_cnt += 1      
+        
+        if (nnp_cnt/total)>=0.8:
+            return True
+        return False
+    except Exception as e:
+        print(e)
+        return True
+    
+df["to_drop"]=df["caption"].apply(drop_no)
+df = df[df["to_drop"]==False]
+df = df.drop("to_drop",axis=1)
+
+df["index_row"] = df.index
+
+df.to_csv(sys.argv[2], sep='\t',index=False)
+

data/scripts/coco.py

@@ -0,0 +1,40 @@
+import json
+import collections
+import logging
+import sys
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing COCO dataset")
+
+with open(annotation_file, "r") as f:
+    annotations = json.load(f)["annotations"]
+
+image_path_to_caption = collections.defaultdict(list)
+for element in annotations:
+    caption = f"{element['caption'].lower().rstrip('.')}"
+    image_path = images_dir + "/%012d.jpg" % (element["image_id"])
+    image_path_to_caption[image_path].append(caption)
+
+lines = []
+for image_path, captions in image_path_to_caption.items():
+    lines.append(json.dumps({"image_path": image_path, "captions": captions}))
+
+train_lines = lines[:-10_001]
+valid_lines = lines[-10_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing COCO dataset done. {len(lines)} images processed.")
+
+# python scripts/coco.py annotations/coco_captions_train2017.json coco_dataset_train.json

data/scripts/flicker8k.py

@@ -0,0 +1,42 @@
+import json
+import logging
+import sys
+import os.path
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing Flicker 8k dataset")
+
+with open(annotation_file, "r") as f:
+    annotations = json.load(f)
+
+lines = []
+for image_path, captions in annotations.items():
+    edited_captions = []
+    for caption in captions:
+        if len(caption) > 0:
+            edited_captions.append(caption.replace("<start> ", "").replace(" <end>", ""))
+    full_image_path =  images_dir+"/"+image_path
+    if os.path.isfile(full_image_path):
+        if len(edited_captions) > 0:
+            lines.append(json.dumps({"image_path": full_image_path, "captions": edited_captions}))
+    else:
+        print(f"{full_image_path} doesn't exist")
+
+train_lines = lines[:-801]
+valid_lines = lines[-801:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing Flicker 8k dataset done. {len(lines)} images processed.")
+

data/scripts/flickr30k.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing Flicker 30k dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+images_dict = {}
+
+for index, caption, image_name in df.itertuples():
+    if image_name in images_dict:
+        images_dict[image_name] += [caption]
+    else:
+        images_dict[image_name] = [caption]
+
+lines = []
+
+for image_path, captions in images_dict.items():
+    full_image_path =  images_dir+"/"+image_name
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": captions}))
+    else:
+        print(f"{full_image_path} doesn't exist")
+
+train_lines = lines[:-3_001]
+valid_lines = lines[-3_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing Flicker 30k dataset done. {len(lines)} images processed.")
+

data/scripts/subcaption.py

@@ -0,0 +1,59 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing subcaption dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+lines = []
+
+df = df[["caption", "url"]]
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+print(f"Loaded {len(df)} images.")
+
+for index, caption_reference_description, image_url in df.itertuples():
+    #index+=1
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{index:08d}---{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        #print(f"{full_image_path} doesn't exist")
+        logging.error(full_image_path)
+
+
+train_lines = lines[:-100_001]
+valid_lines = lines[-100_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing subcaption dataset done. {len(lines)} images processed.")
+

data/scripts/wit.py

@@ -0,0 +1,56 @@
+import pandas as pd
+import os.path
+import sys
+import json
+import logging
+import contexttimer
+import numpy as np
+
+if len(sys.argv) != 4:
+    print("Provide .tsv file name, images dir, output file name. e.g. python coco.py coco_captions_train2017.json /mnt/disks/data-1/flickr8k/coco_train.json coco_dataset_train.json")
+    exit(1)
+
+annotation_file = sys.argv[1]
+images_dir = sys.argv[2]
+output_file = sys.argv[3]
+
+logging.info("Processing WIT dataset")
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(annotation_file, delimiter='\t')
+
+images_dict = {}
+
+lines = []
+
+df = df[["caption_reference_description", "image_url"]]
+
+
+df = df.replace('', np.nan)
+df = df.dropna()
+
+
+
+for index, caption_reference_description, image_url in df.itertuples():
+    base_url = os.path.basename(image_url)  # extract base url
+    stem, ext = os.path.splitext(base_url)  # split into stem and extension
+    filename = f'{stem}.jpg'
+
+    full_image_path =  images_dir+"/"+filename
+
+    if os.path.isfile(full_image_path):
+        lines.append(json.dumps({"image_path": full_image_path, "captions": [caption_reference_description]}))
+    else:
+        print(f"{full_image_path} doesn't exist")
+
+train_lines = lines[:-9_001]
+valid_lines = lines[-9_001:]
+
+with open(output_file+"_train.json", "w") as f:
+    f.write("\n".join(train_lines))
+
+with open(output_file+"_val.json", "w") as f:
+    f.write("\n".join(valid_lines))
+
+logging.info(f"Processing Flicker WIT dataset done. {len(lines)} images processed.")
+

data/scripts/wit_propn.py

@@ -0,0 +1,68 @@
+import sys
+import os
+from datetime import datetime
+import pandas as pd
+import contexttimer
+from urllib.request import urlopen
+import requests
+from PIL import Image
+import torch
+from torchvision.transforms import functional as TF
+from multiprocessing import Pool
+from tqdm import tqdm
+import logging
+import sys
+import numpy as np
+
+
+
+from nltk.tag import CRFTagger
+ct = CRFTagger()
+ct.set_model_file('all_indo_man_tag_corpus_model.crf.tagger')
+
+headers = {
+    "User-Agent": "Googlebot-Image/1.0",  # Pretend to be googlebot
+    "X-Forwarded-For": "64.18.15.200",
+}
+
+# Setup
+logging.basicConfig(filename='download.log', filemode='w', level=logging.INFO)
+requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+
+'''if len(sys.argv) != 3:
+    print("Provide .tsv file name & output directory. e.g. python downloader.py Train-GCC-training.tsv training")
+    exit(1)'''
+
+# Load data
+print(f'Starting to load at {datetime.now().isoformat(timespec="minutes")}')
+
+with contexttimer.Timer(prefix="Loading from tsv"):
+    df = pd.read_csv(sys.argv[1], delimiter='\t')
+    df = df[["caption_reference_description", "image_url"]]
+
+def drop_no(text):
+    try:
+        if len(text)==0:
+            return True
+        text = text.split()
+        result = ct.tag_sents([text])
+        nnp_cnt = 0
+        total = len(result[0])
+
+        for x in result[0]:
+            if x[1] == "NNP":
+                nnp_cnt += 1      
+        
+        if (nnp_cnt/total)>=0.8:
+            return True
+        return False
+    except Exception as e:
+        print(e)
+        return True
+    
+df["to_drop"]=df["caption_reference_description"].apply(drop_no)
+df = df[df["to_drop"]==False]
+df = df.drop("to_drop",axis=1)
+
+df.to_csv(sys.argv[2], sep='\t')
+