Commit 35040880 authored by Grant's avatar Grant
Browse files

ci: dry run

parent f95d1f91

.gitlab-ci.yml

+2 −0
Original line number Diff line number Diff line
@@ -34,6 +34,8 @@ lemmy fetcher: 
  stage: lemmy

  rules:

    - if: $CI_PIPELINE_SOURCE == "schedule"

  variables:

    DRY_RUN: "true"

  before_script:

    - apk add git

    - pip install requests

tools/lemmy_fetcher.py

+62 −30
Original line number Diff line number Diff line
@@ -63,9 +63,10 @@ def logout(): 
	})



class LemmyFetcher:

	def __init__(self, edition):

	def __init__(self, edition, dry_run=False):

		assert edition in EDITIONS

		self.edition = edition

		self.dry_run = dry_run

		self.entry_manager = EntryManager(EDITIONS[self.edition]["entry_folder"])

		self.forge = GitlabForge()

		self.forge.from_env()
@@ -140,6 +141,9 @@ def process_post(self, post_body, make_pr): 
			if self.is_post_failed(post_id, post_body_main_source):

				print("Post already failed with same body: " + str(post_id))

				return

			else:

				if self.dry_run:

					print("DRY RUN: would mark post as failed: " + str(post_id))

				else:

					self.mark_post_as_failed(post_id, post_body_main_source)

				self.send_comment(post_id, "An error occurred while parsing this post. Maybe it’s just not a submission, but if it should, then it is not well formatted.\n\nDetailed error (note that stripping has been done on the JSON content): " + str(ex))
@@ -157,20 +161,34 @@ def process_post(self, post_body, make_pr): 
				raise BaseException("Some change in entries are uncommited, refusing to continue")

			# check new branch does not already exist

			if new_branch_name in subprocess.check_output(["git", "branch", "--list", new_branch_name], text=True):

				print("Branch " + new_branch_name + " already exists, deleting it")

				print("Branch " + new_branch_name + " already exists" + (", would delete it" if self.dry_run else ", deleting it"))

				if not self.dry_run:

					subprocess.check_call(["git", "branch", "-D", new_branch_name])



			if self.dry_run:

				print("DRY RUN: would create branch " + new_branch_name)

			else:

				# create new branch

				subprocess.check_call(["git", "checkout", "-b", new_branch_name])



		success = True

		try:

			if self.dry_run:

				print("DRY RUN: would push entry for post " + str(post_id))

			else:

				self.entry_manager.push_entry(post_json, "lemmy", post_id)

		except Exception as e:

			print("Failed to process post " + str(post_id) + ": " + str(e))

			success = False



		if make_pr:

			if self.dry_run:

				print("DRY RUN: would process PR for post " + str(post_id))

				if success:

					print("DRY RUN: would commit, push branch " + new_branch_name + ", and create PR")

				else:

					print("DRY RUN: would restore entry folder and checkout main")

			else:

				if success:

					subprocess.check_call(["git", "add", EDITIONS[self.edition]["entry_folder"]])

					subprocess.check_call(["git", "commit", "-m", "Add submission from Lemmy post " + str(post_id)])
@@ -186,10 +204,16 @@ def process_post(self, post_body, make_pr): 
					subprocess.check_call(["git", "restore", "--source=HEAD", EDITIONS[self.edition]["entry_folder"]])

				subprocess.check_call(["git", "checkout", "main"])

			self.read_ids.add(str(post_id))

			if self.dry_run:

				print("DRY RUN: would mark post as processed: " + str(post_id))

			else:

				f = open(STATE_FOLDER["processed"] + "/" + str(post_id), "w")

				f.close()



	def send_comment(self, post_id, comment):

		if self.dry_run:

			print("DRY RUN: would comment on post " + str(post_id) + ": " + comment)

			return

		req = requests.post(ENDPOINTS["comment"], json={

			"content": comment,

			"post_id": post_id,
@@ -219,17 +243,25 @@ def crawl_latest(self): 
			except Exception as e:

				print("Failed to process post in an horrible way: " + str(post["post"]["id"]) + " edition " + str(self.edition) + " : " + str(e))

				print(traceback.format_exc())

				if not self.dry_run:

					subprocess.check_call(["git", "checkout", "main"])



if __name__ == "__main__":

	dry_run = bool(os.environ.get("DRY_RUN"))



	if dry_run:

		print("DRY RUN MODE: no changes will be pushed to remote or commented on Lemmy")

	else:

		login()



lm2026 = LemmyFetcher("2026")

	lm2026 = LemmyFetcher("2026", dry_run=dry_run)

	lm2026.crawl_latest()



lm2025 = LemmyFetcher("2025")

	lm2025 = LemmyFetcher("2025", dry_run=dry_run)

	lm2025.crawl_latest()



lm2024 = LemmyFetcher("2024")

	lm2024 = LemmyFetcher("2024", dry_run=dry_run)

	lm2024.crawl_latest()



	if not dry_run:

		logout()
 
 No newline at end of file