Commit 6105a9f2 authored by Grant's avatar Grant
Browse files

Merge branch 'feat/2026' into 'main' 

2026 config & fetcher to ci

See merge request !471
parents 4eea3d05 00e43317

.gitlab-ci.yml

+28 −0
Original line number Diff line number Diff line 
# See https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml



stages:

  - .pre

  - lemmy

  - build

  - test

  - deploy

  - .post



include:

  - local: .gitlab/build-year.yml

    inputs:

      year: 2026

    rules:

      - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

  - local: .gitlab/build-year.yml

    inputs:

      year: 2025
@@ -12,6 +25,18 @@ include: 
    rules:

      - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH



lemmy fetcher:

  image: python:3-alpine

  stage: lemmy

  rules:

    - if: $CI_PIPELINE_SOURCE == "schedule"

  script:

    - python ./tools/lemmy_fetcher.py

  cache:

    key: lemmy-fetcher

    paths:

      - .fetcher



build:

  image: python:3-alpine

  stage: build
@@ -20,12 +45,14 @@ build: 
  script:

    - python ./tools/merge_data.py ./entries/2024 ./atlas2024.json

    - python ./tools/merge_data.py ./entries/2025 ./atlas2025.json

    - python ./tools/merge_data.py ./entries/2026 ./atlas2026.json

  artifacts:

    expire_in: "1 hour"

    expose_as: "atlas_json"

    paths:

      - "atlas2024.json"

      - "atlas2025.json"

      - "atlas2026.json"



comment:

  image: alpine
@@ -41,6 +68,7 @@ comment: 
    - build

  before_script:

    - apk add curl jq git

    - git fetch origin main

  script:

    - |

      # 📝 Build MR Comment Body

entries/2026/.keep

0 → 100644
+0 −0

Empty file added.

tools/lemmy_fetcher.py

+58 −16
Original line number Diff line number Diff line
@@ -8,6 +8,16 @@ 
from lib.gitlab_forge import GitlabForge

import hashlib



# cached in gitlab-ci

STATE_FOLDER = ".fetcher/"



LEMMY_HOST = "toast.ooo"



GIT_CONFIG = [

	"-c", "user.name=Atlas Bot [CI]",

	"-c", "user.email=canvas-atlas@sc07.com"

]



EDITIONS = {

	"2024": {

		"community": "2024lemmycanvasatlas",
@@ -17,12 +27,42 @@ 
		"community": "2025fedicanvasatlas",

		"entry_folder": "entries/2025/"

	},

	"2026": {

		"community": "2026canvasatlas",

		"entry_folder": "entries/2026/"

	},

	"test": {

		"community": "testatlas",

		"entry_folder": "entries/2025/"

	}

}



ENDPOINTS = {

	"login": "https://" + LEMMY_HOST + "/api/v3/user/login",

	"logout": "https://" + LEMMY_HOST + "/api/v3/user/logout",

	"list_posts": "https://" + LEMMY_HOST + "/api/v3/post/list",

	"comment": "https://" + LEMMY_HOST + "/api/v3/comment"

}



JWT = ""



def login():

	global JWT

	login_req = requests.post(ENDPOINTS["login"], json={

		"username_or_email": os.environ.get("LEMMY_USERNAME"),

		"password": os.environ.get("LEMMY_PASSWORD"),

		"totp_2fa_token": "",

	})

	

	assert login_req.status_code == 200

	JWT = login_req.json()["jwt"]



def logout():

	global JWT

	requests.post(ENDPOINTS["logout"], headers={

		"Authorization": "Bearer " + JWT

	})



class LemmyFetcher:

	def __init__(self, edition):

		assert edition in EDITIONS
@@ -31,8 +71,9 @@ def __init__(self, edition): 
		self.forge = GitlabForge()

		self.forge.from_env()

		self.read_ids = set()

		os.makedirs("processed_posts", exist_ok = True)

		os.makedirs("failed_posts", exist_ok = True)

		

		self.prepare_dirs()



		for filename in os.listdir("processed_posts"):

			self.read_ids.add(filename)
@@ -40,15 +81,10 @@ def __init__(self, edition): 
		for filename in os.listdir("failed_posts"):

			self.failed_ids_hashes.add(filename)



		login_req = requests.post("https://toast.ooo/api/v3/user/login", json={

			"username_or_email": os.environ.get("LEMMY_USERNAME"),

			"password": os.environ.get("LEMMY_PASSWORD"),

			"totp_2fa_token": "",

		})

		print(login_req.content)

		assert login_req.status_code == 200

		self.lemmy_jwt = login_req.json()["jwt"]



	def prepare_dirs():

		os.makedirs(STATE_FOLDER, exist_ok=True)

		os.makedirs(STATE_FOLDER + "processed_posts", exist_ok=True)

		os.makedirs(STATE_FOLDER + "failed_posts", exist_ok=True)



	def is_post_failed(self, post_id, body):

		entry_key = str(post_id) + "-" + self.hash_body(body)
@@ -138,7 +174,7 @@ def process_post(self, post_body, make_pr): 
		if make_pr:

			if success:

				subprocess.check_call(["git", "add", EDITIONS[self.edition]["entry_folder"]])

				subprocess.check_call(["git", "-c", "user.name=lemmy_canvas_bot", "-c", "user.email=lemmy_canvas_bot@mariusdavid.fr", "commit", "-m", "Add submission from Lemmy post " + str(post_id)])

				subprocess.check_call(["git", *GIT_CONFIG, "commit", "-m", "Add submission from Lemmy post " + str(post_id)])

				subprocess.check_call(["git", "push", "--force", "origin", new_branch_name])

				pr_url = self.forge.does_pr_already_exist(new_branch_name)

				if pr_url == False:
@@ -155,18 +191,18 @@ def process_post(self, post_body, make_pr): 
			f.close()



	def send_comment(self, post_id, comment):

		req = requests.post("https://toast.ooo/api/v3/comment", json={

		req = requests.post(ENDPOINTS["comment"], json={

			"content": comment,

			"post_id": post_id,

		}, headers = {

			"Authorization": "Bearer " + self.lemmy_jwt

			"Authorization": "Bearer " + JWT

		})

		print(req.content)

		assert req.status_code == 200

		print("Commented on post " + str(post_id))



	def crawl_latest(self):

		req = requests.get("https://toast.ooo/api/v3/post/list", params={

		req = requests.get(ENDPOINTS["list_posts"], params={

			"community_name": EDITIONS[self.edition]["community"],

			"type_": "All",

			"sort": "New",
@@ -177,7 +213,7 @@ def crawl_latest(self): 


		data = req.json()



		print("Fetched " + str(len(data["posts"])) + " post fetched");

		print("Fetched " + str(len(data["posts"])) + " posts");

		for post in data["posts"]:

			try:

				self.process_post(post["post"], True)
@@ -186,9 +222,15 @@ def crawl_latest(self): 
				print(traceback.format_exc())

				subprocess.check_call(["git", "checkout", "main"])



login()



lm2026 = LemmyFetcher("2026")

lm2026.crawl_latest()



lm2025 = LemmyFetcher("2025")

lm2025.crawl_latest()



lm2024 = LemmyFetcher("2024")

lm2024.crawl_latest()



logout()
 
 No newline at end of file

tools/pre_process.py

+3 −1
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ def process_edition(year, community): 
		.replace("{YEAR}", year) \

		.replace("{COMMUNITY}", community)



	for known_year in ["2025", "2024"]:

	for known_year in ["2026", "2025", "2024"]:

		if known_year == year:

			index = index.replace("{" + known_year + "ONLYCOMMENTSTART}", "<!--")

			index = index.replace("{" + known_year + "ONLYCOMMENTEND}", "-->")
@@ -32,6 +32,8 @@ def process_edition(year, community): 
	process_edition("2024", "2024lemmycanvasatlas")

elif sys.argv[1] == "2025":

	process_edition("2025", "2025fedicanvasatlas")

elif sys.argv[1] == "2026":

	process_edition("2026", "2026canvasatlas")

else:

	print("Unrecognizeable edition: " + sys.argv[1])

	exit(-1)

web/_img/fedicanvas2026/final.png

0 → 100644
+154 KiB
Loading image diff...