Commit 6346aacb authored by svc-mlops's avatar svc-mlops

sim: đồng bộ code serving, bump v1.0.2

parent 362333fa
Pipeline #4 passed with stages
in 26 seconds
......@@ -31,6 +31,8 @@ from chatbot_api import (check_service, download_artifact, get_job_log,
TIMEOUT_NLP = 12 * 3600
TIMEOUT_LIGHT = 4 * 3600
# Mọi task ghi multiple_outputs=False: TaskFlow thấy kiểu trả về dict sẽ tự đẩy từng key thành XCom riêng,
# và Airflow 3.0.2 trả 422 với key có giá trị None (vd "error" từ API 106) làm task kẹt rồi fail.
STEP_TASKS = ("train", "export_onnx", "package_tokenizer", "package_json")
PUSH_FILES = {
"model.onnx": "model_repository/PhoBERT/1/model.onnx",
......@@ -102,11 +104,11 @@ def _step(step: str, p: dict, timeout: int, overrides: dict | None = None) -> di
)
def train_pipeline():
@task(task_id="check_106")
@task(task_id="check_106", multiple_outputs=False)
def _check() -> None:
check_service()
@task(task_id="train")
@task(task_id="train", multiple_outputs=False)
def _train(**context) -> dict:
p = context["params"]
ovr = _overrides(p)
......@@ -114,15 +116,15 @@ def train_pipeline():
print(f"override config cho run này: {ovr}")
return _step("nlp", p, TIMEOUT_NLP, ovr)
@task(task_id="export_onnx")
@task(task_id="export_onnx", multiple_outputs=False)
def _onnx(**context) -> dict:
return _step("onnx", context["params"], TIMEOUT_LIGHT)
@task(task_id="evaluate")
@task(task_id="evaluate", multiple_outputs=False)
def _evaluate(**context) -> dict:
return _step("smoke_test", context["params"], TIMEOUT_LIGHT)
@task(task_id="gate")
@task(task_id="gate", multiple_outputs=False)
def _gate(smoke: dict, **context) -> dict:
"""Giữ logic cổng chặn cũ (sprint 2 đổi sang metrics.json + so model prod)."""
p = context["params"]
......@@ -148,15 +150,15 @@ def train_pipeline():
return {"accuracy": acc, "decision": "warn"}
return {"accuracy": acc, "decision": "pass"}
@task(task_id="package_tokenizer")
@task(task_id="package_tokenizer", multiple_outputs=False)
def _tokenizer(**context) -> dict:
return _step("tokenizer", context["params"], TIMEOUT_LIGHT)
@task(task_id="package_json")
@task(task_id="package_json", multiple_outputs=False)
def _json(**context) -> dict:
return _step("json", context["params"], TIMEOUT_LIGHT)
@task(task_id="verify")
@task(task_id="verify", multiple_outputs=False)
def _verify(**context) -> None:
ti = context["ti"]
paths = []
......@@ -174,7 +176,7 @@ def train_pipeline():
raise AirflowException(f"{len(bad)} file thiếu hoặc rỗng: {bad}")
print(f"verify OK: {len(paths)} file")
@task(task_id="register")
@task(task_id="register", multiple_outputs=False)
def _register(**context) -> dict:
"""Ghi lần train vào MLflow và tạo model version alias "candidate"."""
import mlflow
......@@ -228,7 +230,7 @@ def train_pipeline():
raise AirflowException(f"git {args[0]} lỗi: {msg.replace(secret, '***') if secret else msg}")
return r.stdout.strip()
@task(task_id="push_gitlab")
@task(task_id="push_gitlab", multiple_outputs=False)
def _push(**context) -> dict:
"""Như task CI_CD cũ: model.onnx + 4 tokenizer vào repo, bump VERSION, push master -> CI cũ chạy."""
p, ti = context["params"], context["ti"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment