Commit 7deb823f authored by marius david's avatar marius david
Browse files

bot feedback

parent f6b0b940

.gitignore

+5 −1
Original line number Diff line number Diff line 
# Secrets

credentials

praw

secrets.fish



# Temporary files

temp-atlas.json
@@ -44,3 +45,6 @@ venv/ 
ENV/

env.bak/

venv.bak/





processed_posts/
 
 No newline at end of file

tools/entry_manager.py

+1 −31
Original line number Diff line number Diff line
@@ -78,33 +78,3 @@ def is_post_processed(self, source_name, source_id): 
		if source_name in self.combined_source_list:

			return source_id in self.combined_source_list[source_name]

		return False
 
 No newline at end of file
 


"""read_ids = set()

with open("../data/lemmy_read_ids.txt", "r") as f:

	for line in f.readlines():

		if line.strip() == "":

			continue

		read_ids.add(int(line.strip()))



highest_post_id = max(read_ids) or 0



next_page = None



while True:

	req = requests.get("https://toast.ooo/api/v3/post/list", params={

		"next_page": next_page,

		"community_name": "2024lemmycanvasatlas",

		"type_": "All",

		"sort": "New",

	})

	if req.status_code != 200:

		raise BaseException("Request failed with code " + str(req.status_code))



	data = req.json()



	for post in data["posts"]:

		print(post)



		if post["i"]

	

	next_page = data["next_page"] """
 
 No newline at end of file

tools/lemmy_fetcher.py

+80 −14
Original line number Diff line number Diff line 
import json

import subprocess

import requests

import os

import traceback

from entry_manager import EntryManager

from lib.github_forge import GithubForge
@@ -8,14 +11,37 @@ def __init__(self, entry_manager): 
		self.entry_manager = entry_manager

		self.forge = GithubForge()

		self.forge.from_env()

		self.read_ids = set()

		os.makedirs("processed_posts", exist_ok = True)

		for filename in os.listdir("processed_posts"):

			self.read_ids.add(filename)

		

		# from env variable variable

		login_req = requests.post("https://toast.ooo/api/v3/user/login", json={

			"username_or_email": os.environ.get("LEMMY_USERNAME"),

			"password": os.environ.get("LEMMY_PASSWORD"),

			"totp_2fa_token": "",

		})

		print(login_req.content)

		assert login_req.status_code == 200

		self.lemmy_jwt = login_req.json()["jwt"]

	

	

	def process_post(self, post_body, make_pr = False):

		post_id = post_body["id"]

		if self.entry_manager.is_post_processed("lemmy", post_id):



		"""if str(post_id) in self.read_ids:

			print("Post already processed: " + str(post_id))

			return

		

		if self.entry_manager.is_post_processed("lemmy", post_id):

			print("Post already used in an entry: " + str(post_id))

			return"""

		

		if "body" not in post_body:

			print("Post does not contain body: " + str(post_id))

			return

		

		post_body = post_body["body"]



		post_json = None
@@ -23,12 +49,11 @@ def process_post(self, post_body, make_pr = False): 
			post_json = json.loads(post_body)

		except:

			print("Post does not contain valid JSON: " + str(post_id))

			self.send_comment(post_id, "An error occurred while parsing this post. Maybe it’s just not a submission, but if it should, then it is not well formatted.")

			return

		

		new_branch_name = "lemmy-" + str(post_id)

		if make_pr:

			if self.forge.does_pr_already_exist(new_branch_name):

				raise BaseException("PR already exists for this branch, refusing to continue: " + new_branch_name)

			# Yes, I used AI assisted tooling. And they work pretty well. Thanks codeium.

			# get git branch name

			branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], text=True).strip()
@@ -45,23 +70,64 @@ def process_post(self, post_body, make_pr = False): 
			# create new branch

			subprocess.check_call(["git", "checkout", "-b", new_branch_name])



		success = True

		try:

			self.entry_manager.push_entry(post_json, "lemmy", post_id)

		except Exception as e:

			print("Failed to process post " + str(post_id) + ": " + str(e))

			success = False



		if make_pr:

			if success:

				subprocess.check_call(["git", "add", "entries/"])

				subprocess.check_call(["git", "commit", "-m", "Add submission from Lemmy post " + str(post_id)])

				subprocess.check_call(["git", "push", "--force", "origin", new_branch_name])

				pr_url = self.forge.does_pr_already_exist(new_branch_name)

				if pr_url == False:

					print("branch pushed, making PR")

			self.forge.make_pr_between_branches(new_branch_name, "main", "Add submission from Lemmy post " + str(post_id), "Add submission from Lemmy post " + str(post_id))

					pr_url = self.forge.make_pr_between_branches(new_branch_name, "main", "Add submission from Lemmy post " + str(post_id), "An automated PR for submission " + str(post_id))

				else:

					print("branch pushed, PR already exists")

				self.send_comment(post_id, "Thanks for your submission! The automatic PR has been opened at: " + pr_url)

			else:

				subprocess.check_call(["git", "restore", "--source=HEAD", "entries/"])

			subprocess.check_call(["git", "checkout", "main"])

			self.read_ids.add(str(post_id))

			f = open("processed_posts/" + str(post_id), "w")

			f.close()

	

	def send_comment(self, post_id, comment):

		req = requests.post("https://toast.ooo/api/v3/comment", json={

			"content": comment,

			"post_id": post_id,

		}, headers = {

			"Authorization": "Bearer " + self.lemmy_jwt

		})

		print(req.content)

		assert req.status_code == 200

		print("Commented on post " + str(post_id))



	def crawl_latest(self):

		req = requests.get("https://toast.ooo/api/v3/post/list", params={

			"community_name": "2024lemmycanvasatlas",

			"type_": "All",

			"sort": "New",

			"limit": 50

		})

		if req.status_code != 200:

			raise BaseException("Request failed with code " + str(req.status_code))

		

		data = req.json()



		for post in data["posts"]:

			try:

				self.process_post(post["post"], True)

			except Exception as e:

				print("Failed to process post in an horrible way: " + str(post["post"]["id"]) + ": " + str(e))

				print(traceback.format_exc())

				subprocess.check_call(["git", "checkout", "main"])











em = EntryManager()

lm = LemmyFetcher(em)



lm.process_post(json.load(open("tools/test_post.json", "r"))["posts"][0]["post"], False)



print("done")
 
 No newline at end of file
 
lm.crawl_latest()
 
 No newline at end of file

tools/lib/github_forge.py

+3 −3
Original line number Diff line number Diff line
@@ -29,16 +29,16 @@ def does_pr_already_exist(self, from_branch): 


		print(self.repo_name + ":" + from_branch)

		for pr in self.repo.get_pulls(state="open", head=from_branch):

			return True

			return pr.html_url



		return False

	

	def make_pr_between_branches(self, from_branch, to_branch, title, body):

		self.load_if_needed()



		self.repo.create_pull(

		return self.repo.create_pull(

			title = title,

			base = to_branch,

			head = from_branch,

			body = body

		)
 
 No newline at end of file
 
		).html_url
 
 No newline at end of file