Commit fc1979fa authored by Grant's avatar Grant
Browse files

fetcher: move consts

parent 1ce41f62

tools/lemmy_fetcher.py

+12 −8
Original line number Diff line number Diff line
@@ -9,7 +9,11 @@ 
import hashlib



# cached in gitlab-ci

STATE_FOLDER = ".fetcher/"

STATE_FOLDER = {

	"ROOT": ".fetcher/",

	"processed": ".fetcher/processed_posts",

	"failed": ".fetcher/failed_posts"

}



LEMMY_HOST = "toast.ooo"
@@ -74,17 +78,17 @@ def __init__(self, edition): 
		

		self.prepare_dirs()



		for filename in os.listdir("processed_posts"):

		for filename in os.listdir(STATE_FOLDER["processed"]):

			self.read_ids.add(filename)



		self.failed_ids_hashes = set()

		for filename in os.listdir("failed_posts"):

		for filename in os.listdir(STATE_FOLDER["failed"]):

			self.failed_ids_hashes.add(filename)



	def prepare_dirs(self):

		os.makedirs(STATE_FOLDER, exist_ok=True)

		os.makedirs(STATE_FOLDER + "processed_posts", exist_ok=True)

		os.makedirs(STATE_FOLDER + "failed_posts", exist_ok=True)

		os.makedirs(STATE_FOLDER["ROOT"], exist_ok=True)

		os.makedirs(STATE_FOLDER["processed"], exist_ok=True)

		os.makedirs(STATE_FOLDER["failed"], exist_ok=True)



	def is_post_failed(self, post_id, body):

		entry_key = str(post_id) + "-" + self.hash_body(body)
@@ -93,7 +97,7 @@ def is_post_failed(self, post_id, body): 
	def mark_post_as_failed(self, post_id, body):

		entry_key = str(post_id) + "-" + self.hash_body(body)

		self.failed_ids_hashes.add(entry_key)

		with open("failed_posts/" + entry_key, "w") as f:

		with open(STATE_FOLDER["failed"] + "/" + entry_key, "w") as f:

			f.write(body)



	def hash_body(self, body):
@@ -187,7 +191,7 @@ def process_post(self, post_body, make_pr): 
				subprocess.check_call(["git", "restore", "--source=HEAD", EDITIONS[self.edition]["entry_folder"]])

			subprocess.check_call(["git", "checkout", "main"])

			self.read_ids.add(str(post_id))

			f = open("processed_posts/" + str(post_id), "w")

			f = open(STATE_FOLDER["processed"] + "/" + str(post_id), "w")

			f.close()



	def send_comment(self, post_id, comment):