Commit 6346aacb authored by svc-mlops's avatar svc-mlops

sim: đồng bộ code serving, bump v1.0.2

parent 362333fa
Pipeline #4 passed with stages
in 26 seconds
...@@ -31,6 +31,8 @@ from chatbot_api import (check_service, download_artifact, get_job_log, ...@@ -31,6 +31,8 @@ from chatbot_api import (check_service, download_artifact, get_job_log,
TIMEOUT_NLP = 12 * 3600 TIMEOUT_NLP = 12 * 3600
TIMEOUT_LIGHT = 4 * 3600 TIMEOUT_LIGHT = 4 * 3600
# Mọi task ghi multiple_outputs=False: TaskFlow thấy kiểu trả về dict sẽ tự đẩy từng key thành XCom riêng,
# và Airflow 3.0.2 trả 422 với key có giá trị None (vd "error" từ API 106) làm task kẹt rồi fail.
STEP_TASKS = ("train", "export_onnx", "package_tokenizer", "package_json") STEP_TASKS = ("train", "export_onnx", "package_tokenizer", "package_json")
PUSH_FILES = { PUSH_FILES = {
"model.onnx": "model_repository/PhoBERT/1/model.onnx", "model.onnx": "model_repository/PhoBERT/1/model.onnx",
...@@ -102,11 +104,11 @@ def _step(step: str, p: dict, timeout: int, overrides: dict | None = None) -> di ...@@ -102,11 +104,11 @@ def _step(step: str, p: dict, timeout: int, overrides: dict | None = None) -> di
) )
def train_pipeline(): def train_pipeline():
@task(task_id="check_106") @task(task_id="check_106", multiple_outputs=False)
def _check() -> None: def _check() -> None:
check_service() check_service()
@task(task_id="train") @task(task_id="train", multiple_outputs=False)
def _train(**context) -> dict: def _train(**context) -> dict:
p = context["params"] p = context["params"]
ovr = _overrides(p) ovr = _overrides(p)
...@@ -114,15 +116,15 @@ def train_pipeline(): ...@@ -114,15 +116,15 @@ def train_pipeline():
print(f"override config cho run này: {ovr}") print(f"override config cho run này: {ovr}")
return _step("nlp", p, TIMEOUT_NLP, ovr) return _step("nlp", p, TIMEOUT_NLP, ovr)
@task(task_id="export_onnx") @task(task_id="export_onnx", multiple_outputs=False)
def _onnx(**context) -> dict: def _onnx(**context) -> dict:
return _step("onnx", context["params"], TIMEOUT_LIGHT) return _step("onnx", context["params"], TIMEOUT_LIGHT)
@task(task_id="evaluate") @task(task_id="evaluate", multiple_outputs=False)
def _evaluate(**context) -> dict: def _evaluate(**context) -> dict:
return _step("smoke_test", context["params"], TIMEOUT_LIGHT) return _step("smoke_test", context["params"], TIMEOUT_LIGHT)
@task(task_id="gate") @task(task_id="gate", multiple_outputs=False)
def _gate(smoke: dict, **context) -> dict: def _gate(smoke: dict, **context) -> dict:
"""Giữ logic cổng chặn cũ (sprint 2 đổi sang metrics.json + so model prod).""" """Giữ logic cổng chặn cũ (sprint 2 đổi sang metrics.json + so model prod)."""
p = context["params"] p = context["params"]
...@@ -148,15 +150,15 @@ def train_pipeline(): ...@@ -148,15 +150,15 @@ def train_pipeline():
return {"accuracy": acc, "decision": "warn"} return {"accuracy": acc, "decision": "warn"}
return {"accuracy": acc, "decision": "pass"} return {"accuracy": acc, "decision": "pass"}
@task(task_id="package_tokenizer") @task(task_id="package_tokenizer", multiple_outputs=False)
def _tokenizer(**context) -> dict: def _tokenizer(**context) -> dict:
return _step("tokenizer", context["params"], TIMEOUT_LIGHT) return _step("tokenizer", context["params"], TIMEOUT_LIGHT)
@task(task_id="package_json") @task(task_id="package_json", multiple_outputs=False)
def _json(**context) -> dict: def _json(**context) -> dict:
return _step("json", context["params"], TIMEOUT_LIGHT) return _step("json", context["params"], TIMEOUT_LIGHT)
@task(task_id="verify") @task(task_id="verify", multiple_outputs=False)
def _verify(**context) -> None: def _verify(**context) -> None:
ti = context["ti"] ti = context["ti"]
paths = [] paths = []
...@@ -174,7 +176,7 @@ def train_pipeline(): ...@@ -174,7 +176,7 @@ def train_pipeline():
raise AirflowException(f"{len(bad)} file thiếu hoặc rỗng: {bad}") raise AirflowException(f"{len(bad)} file thiếu hoặc rỗng: {bad}")
print(f"verify OK: {len(paths)} file") print(f"verify OK: {len(paths)} file")
@task(task_id="register") @task(task_id="register", multiple_outputs=False)
def _register(**context) -> dict: def _register(**context) -> dict:
"""Ghi lần train vào MLflow và tạo model version alias "candidate".""" """Ghi lần train vào MLflow và tạo model version alias "candidate"."""
import mlflow import mlflow
...@@ -228,7 +230,7 @@ def train_pipeline(): ...@@ -228,7 +230,7 @@ def train_pipeline():
raise AirflowException(f"git {args[0]} lỗi: {msg.replace(secret, '***') if secret else msg}") raise AirflowException(f"git {args[0]} lỗi: {msg.replace(secret, '***') if secret else msg}")
return r.stdout.strip() return r.stdout.strip()
@task(task_id="push_gitlab") @task(task_id="push_gitlab", multiple_outputs=False)
def _push(**context) -> dict: def _push(**context) -> dict:
"""Như task CI_CD cũ: model.onnx + 4 tokenizer vào repo, bump VERSION, push master -> CI cũ chạy.""" """Như task CI_CD cũ: model.onnx + 4 tokenizer vào repo, bump VERSION, push master -> CI cũ chạy."""
p, ti = context["params"], context["ti"] p, ti = context["params"], context["ti"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment