From c7c892ae4eeae341fb035bfa6ad717ef25a1af65 Mon Sep 17 00:00:00 2001
From: chavinlo <85657083+chavinlo@users.noreply.github.com>
Date: Sun, 11 Sep 2022 11:44:55 -0500
Subject: [PATCH] moved scripts and added nsfw extractors

---
 danbooru_data/local/convert.py                | 19 ++++++++
 .../local/extractfromjson_danboo21.py         |  0
 .../local/nsfw_processer_danboo21.py          | 48 +++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 danbooru_data/local/convert.py
 rename scripts/danbooru21_extract.py => danbooru_data/local/extractfromjson_danboo21.py (100%)
 create mode 100644 danbooru_data/local/nsfw_processer_danboo21.py

diff --git a/danbooru_data/local/convert.py b/danbooru_data/local/convert.py
new file mode 100644
index 0000000..6c01bf0
--- /dev/null
+++ b/danbooru_data/local/convert.py
@@ -0,0 +1,19 @@
+#resizes and adds a black bar to all images in directory original
+
+from PIL import Image, ImageOps
+
+import os
+
+directory = 'original'
+
+for filename in os.listdir(directory):
+    var1 = directory + '/' + filename
+    os.mkdir('E:/convert/original/' + filename)
+    for i in os.listdir(var1):
+        var4 = var1 + '/'
+        var2 = var1 + '/' + i
+        if os.path.isfile(var2):
+           print(var2)
+           im = Image.open(var2)
+           im = ImageOps.pad(im, (512, 512), color='black')
+           im.save('E:/convert/' + var2)
diff --git a/scripts/danbooru21_extract.py b/danbooru_data/local/extractfromjson_danboo21.py
similarity index 100%
rename from scripts/danbooru21_extract.py
rename to danbooru_data/local/extractfromjson_danboo21.py
diff --git a/danbooru_data/local/nsfw_processer_danboo21.py b/danbooru_data/local/nsfw_processer_danboo21.py
new file mode 100644
index 0000000..53e7771
--- /dev/null
+++ b/danbooru_data/local/nsfw_processer_danboo21.py
@@ -0,0 +1,48 @@
+# with open("nsfw-ids.txt", 'r', encoding="utf8") as nsfwfile:
+#     nsfw_list = list(nsfwfile)
+import tqdm
+# ##Read line
+# current_saved_file_count = 0
+# current_line_count = 0
+# for line in nsfw_list:
+#     print(line)
+#     last3_line_raw = line[-4:]
+#     last3_line = last3_line_raw.zfill(4)
+#     print(last3_line_raw)
+#     print(last3_line)
+
+def file_len(filename):
+    with open(filename) as f:
+        for i, _ in enumerate(f):
+            pass
+    return i + 1
+
+def writetofile(input):
+    f = open("files2download.txt", "a")
+    f.write(input + "\n")
+    f.close()
+
+#converts nsfw-ids.txt entries to rsync readable file
+
+with open("nsfw-ids.txt", 'r', encoding="utf8") as nsfwfile:
+    nsfw_list = list(nsfwfile)
+count = 0
+linescount = file_len("nsfw-ids.txt")
+
+##Read line
+for line in nsfw_list:
+    line = line.strip()
+    # print(line)
+    linefilled1 = line.zfill(4)
+    linelast3 = linefilled1[-3:]
+    linedirectory = linelast3.zfill(4)
+    # print("line: " + ">>"+ line + "<<")
+    # print("Linefilled1: " + linefilled1)
+    # print("linelast3: " + linelast3)
+    # print("linedirectory: " + linedirectory)
+    directory = "original/" + linedirectory + "/" + line + ".jpg"
+    # print(directory)
+    # print(directory2)
+    writetofile(directory)
+    count = count + 1
+    print(str(count) + "/" + str(linescount))