Commit f265bc48 authored by Grant's avatar Grant
Browse files

fetcher: extract git into lib

parent 233f5e4c

tools/lemmy_fetcher.py

+29 −40
Original line number Diff line number Diff line 
import json

import datetime

import subprocess

import requests

import os

import traceback

from entry_manager import EntryManager

#from lib.github_forge import GithubForge

from lib.gitlab_forge import GitlabForge

from lib.git_utils import GitUtils

import hashlib



# cached in gitlab-ci
@@ -70,6 +71,7 @@ def __init__(self, edition, dry_run=False): 
		self.entry_manager = EntryManager(EDITIONS[self.edition]["entry_folder"])

		self.forge = GitlabForge()

		self.forge.from_env()

		self.git = GitUtils(dry_run=dry_run)

		self.read_ids = set()

		

		self.prepare_dirs()
@@ -91,6 +93,10 @@ def is_post_failed(self, post_id, body): 
		return entry_key in self.failed_ids_hashes



	def mark_post_as_failed(self, post_id, body):

		if self.dry_run:

			print("DRY RUN: would mark post as processed: " + str(post_id))

			return

		

		entry_key = str(post_id) + "-" + self.hash_body(body)

		self.failed_ids_hashes.add(entry_key)

		with open(STATE_FOLDER["failed"] + "/" + entry_key, "w") as f:
@@ -149,9 +155,6 @@ def process_post(self, post_body, make_pr): 
			if self.is_post_failed(post_id, post_body_main_source):

				print("Post already failed with same body: " + str(post_id))

				return

			else:

				if self.dry_run:

					print("DRY RUN: would mark post as failed: " + str(post_id))

			else:

				self.mark_post_as_failed(post_id, post_body_main_source)

				self.send_comment(post_id, "An error occurred while parsing this post. Maybe it’s just not a submission, but if it should, then it is not well formatted.\n\nDetailed error (note that stripping has been done on the JSON content): " + str(ex))
@@ -163,16 +166,10 @@ def process_post(self, post_body, make_pr): 
			if EDITIONS[self.edition]["entry_folder"] in subprocess.check_output(["git", "status", "--porcelain"], text=True):

				raise BaseException("Some change in entries are uncommited, refusing to continue")

			# check new branch does not already exist

			if new_branch_name in subprocess.check_output(["git", "branch", "--list", new_branch_name], text=True):

				print("Branch " + new_branch_name + " already exists" + (", would delete it" if self.dry_run else ", deleting it"))

				if not self.dry_run:

					subprocess.check_call(["git", "branch", "-D", new_branch_name])

			if new_branch_name in self.git.list_branches(new_branch_name):

				self.git.delete_branch(new_branch_name)



			if self.dry_run:

				print("DRY RUN: would create branch " + new_branch_name)

			else:

				# create new branch

				subprocess.check_call(["git", "checkout", "-b", new_branch_name])

			self.git.create_branch(new_branch_name)



		success = True

		try:
@@ -185,17 +182,10 @@ def process_post(self, post_body, make_pr): 
			success = False



		if make_pr:

			if self.dry_run:

				print("DRY RUN: would process PR for post " + str(post_id))

				if success:

					print("DRY RUN: would commit, push branch " + new_branch_name + ", and create PR")

				else:

					print("DRY RUN: would restore entry folder and checkout main")

			else:

			if success:

					subprocess.check_call(["git", "add", EDITIONS[self.edition]["entry_folder"]])

					subprocess.check_call(["git", "commit", "-m", "Add submission from Lemmy post " + str(post_id)])

					subprocess.check_call(["git", "push", "--force", "origin", new_branch_name])

				self.git.add(EDITIONS[self.edition]["entry_folder"])

				self.git.commit("Add submission from Lemmy post " + str(post_id))

				self.git.force_push(new_branch_name)

				pr_url = self.forge.does_pr_already_exist(new_branch_name)

				if pr_url == False:

					print("branch pushed, making PR")
@@ -204,8 +194,8 @@ def process_post(self, post_body, make_pr): 
					print("branch pushed, PR already exists")

				self.send_comment(post_id, "Thanks for your submission! The automatic PR has been opened at: " + pr_url)

			else:

					subprocess.check_call(["git", "restore", "--source=HEAD", EDITIONS[self.edition]["entry_folder"]])

				subprocess.check_call(["git", "checkout", "main"])

				self.git.restore(EDITIONS[self.edition]["entry_folder"])

			self.git.checkout("main")

			self.read_ids.add(str(post_id))

			self.mark_post_as_processed(post_id)
@@ -213,6 +203,7 @@ def send_comment(self, post_id, comment): 
		if self.dry_run:

			print("DRY RUN: would comment on post " + str(post_id) + ": " + comment)

			return

		

		req = requests.post(ENDPOINTS["comment"], json={

			"content": comment,

			"post_id": post_id,
@@ -253,14 +244,12 @@ def crawl_latest(self): 
	else:

		login()



	lm2026 = LemmyFetcher("2026", dry_run=dry_run)

	lm2026.crawl_latest()



	lm2025 = LemmyFetcher("2025", dry_run=dry_run)

	lm2025.crawl_latest()

	years = [x.strip() for x in os.environ.get("FETCH_YEARS", str(datetime.now().year)).split(',')]

	print("Running for years: " + str(years))



	lm2024 = LemmyFetcher("2024", dry_run=dry_run)

	lm2024.crawl_latest()

	for year in years:

		print("Fetching " + year + "...")

		LemmyFetcher(year, dry_run=dry_run).crawl_latest()



	if not dry_run:

		logout()
 
 No newline at end of file

tools/lib/git_utils.py

0 → 100644
+64 −0
Original line number Diff line number Diff line 
import subprocess



class GitUtils():

	def __init__(self, dry_run: bool):

		self.dry_run = dry_run



	def list_branches(self, name: str):

		return subprocess.check_output(["git", "branch", "--list", name], text=True)

		

	def delete_branch(self, name: str):

		if self.dry_run:

			print("DRY RUN: Would delete branch " + name)

			return

		

		print("Deleting branch " + name + "...")

		subprocess.check_call(["git", "branch", "-D", name])



	def create_branch(self, name: str):

		if self.dry_run:

			print("DRY RUN: Would create branch " + name)

			return

		

		print("Creating branch " + name + "...")

		subprocess.check_call(["git", "checkout", "-b", name])



	def add(self, filename: str):

		if self.dry_run:

			print("DRY RUN: git add " + filename)

			return

		

		print("git: add " + filename)

		subprocess.check_call(["git", "add", filename])



	def commit(self, message: str):

		if self.dry_run:

			print("DRY RUN: git commit -m " + message)

			return



		print("git: commit -m " + message)

		subprocess.check_call(["git", "commit", "-m", message])



	def force_push(self, branch: str):

		if self.dry_run:

			print("DRY RUN: git push --force-with-lease origin " + branch)

			return

		

		print("git: force push " + branch)

		subprocess.check_call(["git", "push", "--force-with-lease", "origin", branch])



	def restore(self, filename: str):

		if self.dry_run:

			print("DRY RUN: git restore " + filename)

			return

		

		print("git: restore " + filename)

		subprocess.check_call(["git", "restore", "--source=HEAD", filename])



	def checkout(self, branch: str):

		if self.dry_run:

			print("DRY RUN: git checkout " + branch)

			return

		

		print("git: checkout" + branch)

		subprocess.check_call(["git", "checkout", branch])