GIT_HEAD_REF := $(shell git rev-parse HEAD) BASE_IMAGE := pytorch/pytorch:2.0.1-cuda11.7-cudnn8-devel DEV_IMAGE_NAME := text-to-sql-dev TRAIN_IMAGE_NAME := text-to-sql-train EVAL_IMAGE_NAME := text-to-sql-eval BUILDKIT_IMAGE := tscholak/text-to-sql-buildkit:buildx-stable-1 BUILDKIT_BUILDER ?= buildx-local BASE_DIR := $(shell pwd) .PHONY: init-buildkit init-buildkit: docker buildx create \ --name buildx-local \ --driver docker-container \ --driver-opt image=$(BUILDKIT_IMAGE),network=host \ --use .PHONY: del-buildkit del-buildkit: docker buildx rm buildx-local .PHONY: build-thrift-code build-thrift-code: thrift1 --gen mstch_cpp2 picard.thrift thrift1 --gen mstch_py3 picard.thrift cd gen-py3 && python setup.py build_ext --inplace .PHONY: build-picard-deps build-picard-deps: cabal update thrift-compiler --hs --use-hash-map --use-hash-set --gen-prefix gen-hs -o . picard.thrift patch -p 1 -N -d third_party/hsthrift < ./fb-util-cabal.patch || true cd third_party/hsthrift \ && make THRIFT_COMPILE=thrift-compiler thrift-cpp thrift-hs cabal build --only-dependencies lib:picard .PHONY: build-picard build-picard: cabal install --overwrite-policy=always --install-method=copy exe:picard .PHONY: build-dev-image build-dev-image: ssh-add docker buildx build \ --builder $(BUILDKIT_BUILDER) \ --ssh default=$(SSH_AUTH_SOCK) \ -f Dockerfile \ --tag yizhangjustin/$(DEV_IMAGE_NAME):$(GIT_HEAD_REF) \ --tag yizhangjustin/$(DEV_IMAGE_NAME):cache \ --tag yizhangjustin/$(DEV_IMAGE_NAME):devcontainer \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --target dev \ --cache-from type=registry,ref=yizhangjustin/$(DEV_IMAGE_NAME):cache \ --cache-to type=inline \ --push \ git@github.com:yizhang-unifr/picard #$(GIT_HEAD_REF) .PHONY: pull-dev-image pull-dev-image: docker pull yizhangjustin/$(DEV_IMAGE_NAME):$(GIT_HEAD_REF) .PHONY: build-train-image build-train-image: ssh-add docker buildx build \ --builder $(BUILDKIT_BUILDER) \ --ssh default=$(SSH_AUTH_SOCK) \ -f Dockerfile \ --tag yizhangjustin/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \ --tag yizhangjustin/$(TRAIN_IMAGE_NAME):cache \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --target train \ --cache-from type=registry,ref=yizhangjustin/$(TRAIN_IMAGE_NAME):cache \ --cache-to type=inline \ --progress=plain \ --verbose \ --push \ git@github.com:yizhang-unifr/picard #$(GIT_HEAD_REF) .PHONY: pull-train-image pull-train-image: docker pull yizhangjustin/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) .PHONY: build-eval-image build-eval-image: ssh-add docker buildx build \ --builder $(BUILDKIT_BUILDER) \ --ssh default=$(SSH_AUTH_SOCK) \ -f Dockerfile \ --tag yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \ --tag yizhangjustin/$(EVAL_IMAGE_NAME):cache \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ --target eval \ --cache-from type=registry,ref=yizhangjustin/$(EVAL_IMAGE_NAME):cache \ --cache-to type=inline \ --no-cache \ --push \ git@github.com:yizhang-unifr/picard#$(GIT_HEAD_REF) .PHONY: pull-eval-image pull-eval-image: docker pull yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) .PHONY: train train: pull-train-image mkdir -p -m 777 train mkdir -p -m 777 transformers_cache mkdir -p -m 777 wandb docker run \ -it \ --rm \ --user 13011:13011 \ --mount type=bind,source=$(BASE_DIR)/train,target=/train \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ --mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \ yizhangjustin/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/run_seq2seq.py configs/train.json" .PHONY: train_cosql train_cosql: pull-train-image mkdir -p -m 777 train mkdir -p -m 777 transformers_cache mkdir -p -m 777 wandb docker run \ -it \ --rm \ --user 13011:13011 \ --mount type=bind,source=$(BASE_DIR)/train,target=/train \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ --mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \ yizhangjustin/$(TRAIN_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/run_seq2seq.py configs/train_cosql.json" .PHONY: eval eval: pull-eval-image mkdir -p -m 777 eval mkdir -p -m 777 transformers_cache mkdir -p -m 777 wandb docker run \ -it \ --rm \ --user 13011:13011 \ --mount type=bind,source=$(BASE_DIR)/eval,target=/eval \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ --mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \ yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/run_seq2seq.py configs/eval.json" .PHONY: eval_cosql eval_cosql: pull-eval-image mkdir -p -m 777 eval mkdir -p -m 777 transformers_cache mkdir -p -m 777 wandb docker run \ -it \ --rm \ --user 13011:13011 \ --mount type=bind,source=$(BASE_DIR)/eval,target=/eval \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ --mount type=bind,source=$(BASE_DIR)/wandb,target=/app/wandb \ yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/run_seq2seq.py configs/eval_cosql.json" .PHONY: serve serve: pull-eval-image mkdir -p -m 777 database mkdir -p -m 777 transformers_cache docker run \ -it \ --rm \ --user 13011:13011 \ -p 8000:8000 \ --mount type=bind,source=$(BASE_DIR)/database,target=/database \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/serve_seq2seq.py configs/serve.json" .PHONY: prediction_output prediction_output: pull-eval-image mkdir -p -m 777 prediction_output docker run \ -it \ --rm \ --user 13011:13011 \ -p 8000:8000 \ --mount type=bind,source=$(BASE_DIR)/prediction_output,target=/prediction_output \ --mount type=bind,source=$(BASE_DIR)/transformers_cache,target=/transformers_cache \ --mount type=bind,source=$(BASE_DIR)/configs,target=/app/configs \ yizhangjustin/$(EVAL_IMAGE_NAME):$(GIT_HEAD_REF) \ /bin/bash -c "python seq2seq/prediction_output.py configs/prediction_output.json"