Makefile

# /!\ /!\ /!\ /!\ /!\ /!\ /!\ DISCLAIMER /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\
#
# This Makefile is only meant to be used for DEVELOPMENT purpose as we are
# changing the user id that will run in the container.
#
# PLEASE DO NOT USE IT FOR YOUR CI/PRODUCTION/WHATEVER...
#
# /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\ /!\
#
# Note to developers:
#
# While editing this file, please respect the following statements:
#
# 1. Every variable should be defined in the ad hoc VARIABLES section with a
#    relevant subsection
# 2. Every new rule should be defined in the ad hoc RULES section with a
#    relevant subsection depending on the targeted service
# 3. Rules should be sorted alphabetically within their section
# 4. When a rule has multiple dependencies, you should:
#    - duplicate the rule name to add the help string (if required)
#    - write one dependency per line to increase readability and diffs
# 5. .PHONY rule statement should be written after the corresponding rule
# ==============================================================================
# VARIABLES

BOLD := \033[1m
RESET := \033[0m
GREEN := \033[1;32m


# -- Database

DB_HOST            = postgresql
DB_PORT            = 5432

# -- Docker
# Get the current user ID to use for docker run and docker exec commands
DOCKER_UID          = $(shell id -u)
DOCKER_GID          = $(shell id -g)
DOCKER_USER         = $(DOCKER_UID):$(DOCKER_GID)
COMPOSE             = DOCKER_USER=$(DOCKER_USER) docker compose
COMPOSE_E2E         = DOCKER_USER=$(DOCKER_USER) docker compose -f compose.yml -f compose-e2e.yml
COMPOSE_EXEC        = $(COMPOSE) exec
COMPOSE_EXEC_APP    = $(COMPOSE_EXEC) app-dev
COMPOSE_RUN         = $(COMPOSE) run --rm
COMPOSE_RUN_APP     = $(COMPOSE_RUN) app-dev
COMPOSE_RUN_CROWDIN = $(COMPOSE_RUN) crowdin crowdin

# -- Backend
MANAGE              = $(COMPOSE_RUN_APP) python manage.py
MAIL_YARN           = $(COMPOSE_RUN) -w /app/src/mail node yarn

# -- Evals (git metadata read on the host; Docker has no .git mount)
EVAL_GIT_COMMIT     = $(shell git rev-parse HEAD 2>/dev/null)
EVAL_GIT_BRANCH     = $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
EVAL_GIT_DIRTY      = $(shell git status --porcelain 2>/dev/null | grep -q . && echo 1 || echo 0)
EVAL_GIT_ENV        = -e EVAL_GIT_COMMIT=$(EVAL_GIT_COMMIT) -e EVAL_GIT_BRANCH=$(EVAL_GIT_BRANCH) -e EVAL_GIT_DIRTY=$(EVAL_GIT_DIRTY)
COMPOSE_RUN_EVAL    = $(COMPOSE_RUN) $(EVAL_GIT_ENV) app-dev

# -- Frontend
PATH_FRONT          = ./src/frontend
PATH_FRONT_CONVERSATIONS  = $(PATH_FRONT)/apps/conversations

# ==============================================================================
# RULES

default: help

data/objectstorage:
	@mkdir -p data/objectstorage

data/static:
	@mkdir -p data/static

# -- Project

create-env-files: ## Copy the dist env files to env files
create-env-files: \
	env.d/development/common \
	env.d/development/crowdin \
	env.d/development/postgresql \
	env.d/development/kc_postgresql
.PHONY: create-env-files

pre-bootstrap: \
	data/objectstorage \
	data/static \
	create-env-files
.PHONY: pre-bootstrap

post-bootstrap: \
	migrate \
	demo \
	back-i18n-compile \
	mails-install \
	mails-build
.PHONY: post-bootstrap


bootstrap: ## Prepare Docker developmentimages for the project
bootstrap: \
	pre-bootstrap \
	build \
	post-bootstrap \
	run
.PHONY: bootstrap

bootstrap-e2e: ## Prepare Docker production images to be used for e2e tests
bootstrap-e2e: \
	pre-bootstrap \
	build-e2e \
	post-bootstrap \
	run-e2e
.PHONY: bootstrap-e2e

bootstrap-e2e-no-build: ## Same as bootstrap-e2e, for images built elsewhere (CI)
bootstrap-e2e-no-build: \
	pre-bootstrap \
	post-bootstrap \
	run-e2e
.PHONY: bootstrap-e2e-no-build

# -- Docker/compose
build: cache ?=
build: ## build the project containers
	@$(MAKE) build-backend cache=$(cache)
	@$(MAKE) build-frontend cache=$(cache)
.PHONY: build

build-backend: cache ?=
build-backend: ## build the app-dev container
	@$(COMPOSE) build app-dev $(cache)
.PHONY: build-backend

build-frontend: cache ?=
build-frontend: ## build the frontend container
	@$(COMPOSE) build frontend-development $(cache)
.PHONY: build-frontend

build-e2e: cache ?=
build-e2e: ## build the e2e container
	@$(MAKE) build-backend cache=$(cache)
	@$(COMPOSE_E2E) build frontend openmockllm-mistral $(cache)
.PHONY: build-e2e

down: ## stop and remove containers, networks, images, and volumes
	@$(COMPOSE_E2E) down
.PHONY: down

logs: ## display app-dev logs (follow mode)
	@$(COMPOSE) logs -f app-dev
.PHONY: logs

run-backend: ## Start only the backend application and all needed services
	@$(COMPOSE) up --force-recreate -d nginx app-dev celery-dev
.PHONY: run-backend

run-celery: ## Start only the celery worker
	@$(COMPOSE) up --force-recreate -d celery-dev
.PHONY: run-celery

run-frontend: ## Start only the frontend application
	@$(COMPOSE) up --force-recreate -d frontend-development
.PHONY: run-frontend

run: ## start the wsgi (production) and development server
run: 
	@$(MAKE) run-backend
	@$(MAKE) run-frontend
.PHONY: run

create-compose-with-models: ## override the docker-compose file with models
	cp -n compose.with_model.override.yml compose.override.yml
.PHONY: create-compose-with-models

run-e2e: ## start the e2e server
run-e2e:
	@$(MAKE) run-backend
	@$(COMPOSE_E2E) up --force-recreate -d frontend openmockllm-mistral
.PHONY: run-e2e

status: ## an alias for "docker compose ps"
	@$(COMPOSE_E2E) ps
.PHONY: status

stop: ## stop the development server using Docker
	@$(COMPOSE_E2E) stop
.PHONY: stop

# -- Backend

demo: ## flush db then create a demo for load testing purpose
	@$(MAKE) resetdb
	@$(MANAGE) create_demo
.PHONY: demo

# Nota bene: Black should come after isort just in case they don't agree...
lint: ## lint back-end python sources
lint: \
  lint-ruff-format \
  lint-ruff-check \
  lint-pylint
.PHONY: lint

lint-ruff-format: ## format back-end python sources with ruff
	@echo 'lint:ruff-format started…'
	@$(COMPOSE_RUN_APP) ruff format .
.PHONY: lint-ruff-format

lint-ruff-check: ## lint back-end python sources with ruff
	@echo 'lint:ruff-check started…'
	@$(COMPOSE_RUN_APP) ruff check . --fix
.PHONY: lint-ruff-check

lint-pylint: ## lint back-end python sources with pylint only on changed files from main
	@echo 'lint:pylint started…'
	bin/pylint --diff-only=origin/main
.PHONY: lint-pylint

test: ## run project tests
	@$(MAKE) test-back-parallel
.PHONY: test

test-back: ## run back-end tests
	@args="$(filter-out $@,$(MAKECMDGOALS))" && \
	bin/pytest $${args:-${1}}
.PHONY: test-back

test-back-parallel: ## run all back-end tests in parallel
	@args="$(filter-out $@,$(MAKECMDGOALS))" && \
	bin/pytest -n auto $${args:-${1}}
.PHONY: test-back-parallel

makemigrations:  ## run django makemigrations for the conversations project.
	@echo "$(BOLD)Running makemigrations$(RESET)"
	@$(COMPOSE) up -d postgresql
	@$(MANAGE) makemigrations
.PHONY: makemigrations

migrate:  ## run django migrations for the conversations project.
	@echo "$(BOLD)Running migrations$(RESET)"
	@$(COMPOSE) up -d postgresql
	@$(MANAGE) migrate
.PHONY: migrate

superuser: ## Create an admin superuser with password "admin"
	@echo "$(BOLD)Creating a Django superuser$(RESET)"
	@$(MANAGE) createsuperuser --email admin@example.com --password admin
.PHONY: superuser

back-i18n-compile: ## compile the gettext files
	@$(MANAGE) compilemessages --ignore=".venv/**/*"
.PHONY: back-i18n-compile

back-i18n-generate: ## create the .pot files used for i18n
	@$(MANAGE) makemessages -a --keep-pot --all
.PHONY: back-i18n-generate

shell: ## connect to database shell
	@$(MANAGE) shell #_plus
.PHONY: shell

deindex_inactive_collections: ## run the deindex_inactive_collections management command
	@$(MANAGE) deindex_inactive_collections
.PHONY: deindex_inactive_collections

fetch_model_health: ## check the health of the models (usage: make fetch_model_health FETCH_MODEL_HEALTH_ARGS="--provider albert")
	@$(MANAGE) fetch_model_health $(FETCH_MODEL_HEALTH_ARGS)
.PHONY: fetch_model_health

eval: ## run behavioral evals (usage: make eval EVAL_ARGS="--dataset url_hallucination --verbose")
	@$(COMPOSE_RUN_EVAL) python manage.py run_evals $(EVAL_ARGS)
.PHONY: eval

eval-debug: ## run behavioral evals with debugpy on port 5678 (attach VS Code before the command runs)
	@$(COMPOSE) run --rm -p 5678:5678 $(EVAL_GIT_ENV) app-dev python -m debugpy --listen 0.0.0.0:5678 --wait-for-client manage.py run_evals $(EVAL_ARGS)
.PHONY: eval-debug

eval-baseline: ## mark a saved eval run as baseline (usage: make eval-baseline EVAL_ARGS="--run latest")
	@$(MANAGE) create_eval_baseline $(EVAL_ARGS)
.PHONY: eval-baseline

eval-compare: ## compare saved eval runs (usage: make eval-compare EVAL_ARGS="--fail-on-regression")
	@$(MANAGE) compare_evals $(EVAL_ARGS)
.PHONY: eval-compare

eval-dashboard: ## generate the HTML dashboard for saved eval runs
	@$(MANAGE) generate_eval_dashboard
.PHONY: eval-dashboard

eval-reset: ## wipe saved eval runs, baselines, and dashboard (usage: make eval-reset EVAL_ARGS="--keep-baselines")
	@$(MANAGE) reset_evals $(EVAL_ARGS)
.PHONY: eval-reset

# -- Database

dbshell: ## connect to database shell
	docker compose exec app-dev python manage.py dbshell
.PHONY: dbshell

resetdb: FLUSH_ARGS ?=
resetdb: ## flush database and create a superuser "admin"
	@echo "$(BOLD)Flush database$(RESET)"
	@$(MANAGE) flush $(FLUSH_ARGS)
	@${MAKE} superuser
.PHONY: resetdb

env.d/development/common:
	cp -n env.d/development/common.dist env.d/development/common

env.d/development/postgresql:
	cp -n env.d/development/postgresql.dist env.d/development/postgresql

env.d/development/kc_postgresql:
	cp -n env.d/development/kc_postgresql.dist env.d/development/kc_postgresql

# -- Internationalization

env.d/development/crowdin:
	cp -n env.d/development/crowdin.dist env.d/development/crowdin

crowdin-download: ## Download translated message from crowdin
	@$(COMPOSE_RUN_CROWDIN) download -c crowdin/config.yml
.PHONY: crowdin-download

crowdin-download-sources: ## Download sources from Crowdin
	@$(COMPOSE_RUN_CROWDIN) download sources -c crowdin/config.yml
.PHONY: crowdin-download-sources

crowdin-upload: ## Upload source translations to crowdin
	@$(COMPOSE_RUN_CROWDIN) upload sources -c crowdin/config.yml
.PHONY: crowdin-upload

i18n-compile: ## compile all translations
i18n-compile: \
	back-i18n-compile \
	frontend-i18n-compile
.PHONY: i18n-compile

i18n-generate: ## create the .pot files and extract frontend messages
i18n-generate: \
	back-i18n-generate \
	frontend-i18n-generate
.PHONY: i18n-generate

i18n-download-and-compile: ## download all translated messages and compile them to be used by all applications
i18n-download-and-compile: \
  crowdin-download \
  i18n-compile
.PHONY: i18n-download-and-compile

i18n-generate-and-upload: ## generate source translations for all applications and upload them to Crowdin
i18n-generate-and-upload: \
  i18n-generate \
  crowdin-upload
.PHONY: i18n-generate-and-upload


# -- Mail generator

mails-build: ## Convert mjml files to html and text
	@$(MAIL_YARN) build
.PHONY: mails-build

mails-build-html-to-plain-text: ## Convert html files to text
	@$(MAIL_YARN) build-html-to-plain-text
.PHONY: mails-build-html-to-plain-text

mails-build-mjml-to-html:	## Convert mjml files to html and text
	@$(MAIL_YARN) build-mjml-to-html
.PHONY: mails-build-mjml-to-html

mails-install: ## install the mail generator
	@$(MAIL_YARN) install
.PHONY: mails-install

# -- Misc
clean: ## restore repository state as it was freshly cloned
	git clean -idx
.PHONY: clean

help:
	@echo "$(BOLD)conversations Makefile"
	@echo "Please use 'make $(BOLD)target$(RESET)' where $(BOLD)target$(RESET) is one of:"
	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "$(GREEN)%-30s$(RESET) %s\n", $$1, $$2}'
.PHONY: help

# Front
frontend-development-install: ## install the frontend locally
	cd $(PATH_FRONT_CONVERSATIONS) && yarn
.PHONY: frontend-development-install

frontend-lint: ## run the frontend linter
	cd $(PATH_FRONT) && yarn lint
.PHONY: frontend-lint

run-frontend-development: ## Run the frontend in development mode
	#@$(COMPOSE) stop frontend frontend-development
	cd $(PATH_FRONT_CONVERSATIONS) && yarn dev
.PHONY: run-frontend-development

frontend-i18n-extract: ## Extract the frontend translation inside a json to be used for crowdin
	cd $(PATH_FRONT) && yarn i18n:extract
.PHONY: frontend-i18n-extract

frontend-i18n-generate: ## Generate the frontend json files used for crowdin
frontend-i18n-generate: \
	crowdin-download-sources \
	frontend-i18n-extract
.PHONY: frontend-i18n-generate

frontend-i18n-compile: ## Format the crowin json files used deploy to the apps
	cd $(PATH_FRONT) && yarn i18n:deploy
.PHONY: frontend-i18n-compile

# -- K8S
build-k8s-cluster: ## build the kubernetes cluster using kind
	./bin/start-kind.sh
.PHONY: build-k8s-cluster

start-tilt-keycloak: ## start Tilt with local Keycloak auth
	DEV_ENV=dev tilt up --namespace=conversations -f ./bin/Tiltfile
.PHONY: start-tilt-keycloak

start-tilt-proconnect: ## start Tilt with staging ProConnect auth + Docs
	DEV_ENV=dev-proconnect tilt up --namespace=conversations -f ./bin/Tiltfile
.PHONY: start-tilt-proconnect

stop-tilt: ## stop Tilt and leave the kind cluster running
	tilt down --namespace=conversations -f ./bin/Tiltfile
.PHONY: stop-tilt

bump-packages-version: VERSION_TYPE ?= minor
bump-packages-version: ## bump the version of the project - VERSION_TYPE can be "major", "minor", "patch"
	cd ./src/mail && yarn version --no-git-tag-version --$(VERSION_TYPE)
	cd ./src/frontend/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
	cd ./src/frontend/apps/e2e/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
	cd ./src/frontend/apps/conversations/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
	cd ./src/frontend/packages/eslint-config-conversations/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
	cd ./src/frontend/packages/i18n/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
.PHONY: bump-packages-version