Lemmy fetcher: better stripping and error message (89e6c7eb) · Commits · fediverse.events / Canvas Atlas

tools/lemmy_fetcher.py

+14 −2

Original line number	Diff line number	Diff line
		@@ -16,6 +16,10 @@
		"2025": {
		"community": "2025fedicanvasatlas",
		"entry_folder": "entries2025/"
		},
		"test": {
		"community": "testatlas",
		"entry_folder": "entries2025/"
		}
		}

		@@ -84,10 +88,18 @@ def process_post(self, post_body, make_pr):
		post_body_main = post_body_main[3:]
		if post_body_main.endswith("```"):
		post_body_main = post_body_main[:-3]
		if not post_body_main.startswith("{"):
		start_index = post_body_main.find('{')
		if start_index != -1:
		post_body_main = post_body_main[start_index:]
		if not post_body_main.endswith("}"):
		end_index = post_body_main.rfind('}')
		if end_index != -1:
		post_body_main = post_body_main[:end_index + 1]

		try:
		post_json = json.loads(post_body_main)
		except:
		except Exception as ex:
		print("Post does not contain valid JSON: " + str(post_id))

		if self.is_post_failed(post_id, post_body_main_source):
		@@ -95,7 +107,7 @@ def process_post(self, post_body, make_pr):
		return
		else:
		self.mark_post_as_failed(post_id, post_body_main_source)
		self.send_comment(post_id, "An error occurred while parsing this post. Maybe it’s just not a submission, but if it should, then it is not well formatted.")
		self.send_comment(post_id, "An error occurred while parsing this post. Maybe it’s just not a submission, but if it should, then it is not well formatted.\n\nDetailed error (note that stripping has been done on the JSON content): " + str(ex))
		return

		new_branch_name = "lemmy-" + str(post_id)