Commit 879bf9f1 authored by Grant's avatar Grant
Browse files

ci: clean up fetcher

parent 5dcb3a1b

.gitlab-ci.yml

+1 −0
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ lemmy fetcher: 
    - git config --global user.email "canvas-atlas@sc07.com"

    - git remote remove origin || true

    - git remote add origin https://oauth2:${GITLAB_CI_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git

    - git fetch origin

  script:

    - python ./tools/lemmy_fetcher.py

    - tree .fetcher

tools/lemmy_fetcher.py

+9 −10
Original line number Diff line number Diff line
@@ -96,6 +96,14 @@ def mark_post_as_failed(self, post_id, body): 
		with open(STATE_FOLDER["failed"] + "/" + entry_key, "w") as f:

			f.write(body)



	def mark_post_as_processed(self, post_id):

		if self.dry_run:

			print("DRY RUN: would mark post as processed: " + str(post_id))

			return

		

		f = open(STATE_FOLDER["processed"] + "/" + str(post_id), "w")

		f.close()



	def hash_body(self, body):

		return hashlib.md5(body.encode()).hexdigest()
@@ -151,11 +159,6 @@ def process_post(self, post_body, make_pr): 


		new_branch_name = "lemmy-" + str(post_id)

		if make_pr:

			# Yes, I used AI assisted tooling. And they work pretty well. Thanks codeium.

			# get git branch name

			branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], text=True).strip()

			# if branch != "main":

			# 	raise BaseException("Not on main branch, refusing to continue: " + branch)

			# check if any content in entries is dirty

			if EDITIONS[self.edition]["entry_folder"] in subprocess.check_output(["git", "status", "--porcelain"], text=True):

				raise BaseException("Some change in entries are uncommited, refusing to continue")
@@ -204,11 +207,7 @@ def process_post(self, post_body, make_pr): 
					subprocess.check_call(["git", "restore", "--source=HEAD", EDITIONS[self.edition]["entry_folder"]])

				subprocess.check_call(["git", "checkout", "main"])

			self.read_ids.add(str(post_id))

			if self.dry_run:

				print("DRY RUN: would mark post as processed: " + str(post_id))

			else:

				f = open(STATE_FOLDER["processed"] + "/" + str(post_id), "w")

				f.close()

			self.mark_post_as_processed(post_id)



	def send_comment(self, post_id, comment):

		if self.dry_run: