Files

732 lines
28 KiB
Python

import io
from dataclasses import dataclass
from datetime import datetime
from typing import Literal
from fastapi import UploadFile
from openpyxl import load_workbook
from openpyxl.styles import Alignment, PatternFill
from openpyxl.utils import get_column_letter
from pydantic import BaseModel, ValidationError
from sqlalchemy.ext.asyncio import AsyncSession
from common.vo import CrudResponseModel
from exceptions.exception import PermissionException, ServiceException
from module_admin.entity.vo.user_vo import CurrentUserModel
from module_business.dao.master_dao import MasterDao
from module_business.domain.test_batch_scope import TestBatchScope
from module_business.entity.vo.master_vo import (
CustomerModel,
LineModel,
SiteModel,
StrainModel,
TunnelModel,
WarehouseModel,
)
from module_business.service.master_service import MasterResource, MasterService
from module_business.service.pallet_service import PalletService
from utils.excel_util import ExcelUtil
MasterExcelResource = Literal['strain', 'customer', 'site', 'line', 'warehouse', 'tunnel', 'pallet']
ImportOutcome = Literal['add', 'update', 'skip']
_MAX_IMPORT_BYTES = 5 * 1024 * 1024
_MAX_IMPORT_ROWS = 5000
_STATUS_TO_VALUE = {'正常': '0', '停用': '1'}
_VALUE_TO_STATUS = {'0': '正常', '1': '停用'}
@dataclass(frozen=True)
class ExcelSpec:
"""一个基础资料资源的导入导出列定义。"""
label: str
headers: tuple[str, ...]
code_header: str
code_attr: str
name_header: str | None = None
name_attr: str | None = None
address_header: str | None = None
address_attr: str | None = None
parent_header: str | None = None
parent_kind: Literal['customer', 'site'] | None = None
parent_attr: str | None = None
status_attr: str = 'status'
id_attr: str = ''
code_max: int = 64
name_max: int = 100
_SPECS: dict[MasterExcelResource, ExcelSpec] = {
'strain': ExcelSpec(
label='菌种',
headers=('菌种编码', '菌种名称', '状态', '备注'),
code_header='菌种编码',
code_attr='strain_code',
name_header='菌种名称',
name_attr='strain_name',
id_attr='strain_id',
),
'customer': ExcelSpec(
label='客户',
headers=('客户编码', '客户名称', '状态', '备注'),
code_header='客户编码',
code_attr='customer_code',
name_header='客户名称',
name_attr='customer_name',
id_attr='customer_id',
),
'site': ExcelSpec(
label='客户现场',
headers=('客户编码', '现场编码', '现场名称', '现场地址', '状态', '备注'),
code_header='现场编码',
code_attr='site_code',
name_header='现场名称',
name_attr='site_name',
address_header='现场地址',
address_attr='address',
parent_header='客户编码',
parent_kind='customer',
parent_attr='customer_id',
id_attr='site_id',
),
'line': ExcelSpec(
label='产线',
headers=('现场编码', '产线编码', '产线名称', '状态', '备注'),
code_header='产线编码',
code_attr='line_code',
name_header='产线名称',
name_attr='line_name',
parent_header='现场编码',
parent_kind='site',
parent_attr='site_id',
id_attr='line_id',
),
'warehouse': ExcelSpec(
label='仓库',
headers=('仓库编码', '仓库名称', '仓库地址', '状态', '备注'),
code_header='仓库编码',
code_attr='warehouse_code',
name_header='仓库名称',
name_attr='warehouse_name',
address_header='仓库地址',
address_attr='address',
id_attr='warehouse_id',
),
'pallet': ExcelSpec(
label='托盘',
headers=('托盘编号', '状态', '备注'),
code_header='托盘编号',
code_attr='pallet_code',
status_attr='asset_status',
id_attr='pallet_id',
),
'tunnel': ExcelSpec(
label='隧道',
headers=('现场编码', '隧道编号', '隧道名称', '状态', '备注'),
code_header='隧道编号',
code_attr='tunnel_code',
name_header='隧道名称',
name_attr='tunnel_name',
parent_header='现场编码',
parent_kind='site',
parent_attr='site_id',
id_attr='tunnel_id',
),
}
_LISTERS = {
'strain': MasterService.list_strain,
'customer': MasterService.list_customer,
'site': MasterService.list_site,
'line': MasterService.list_line,
'warehouse': MasterService.list_warehouse,
'tunnel': MasterService.list_tunnel,
'pallet': PalletService.list_pallet,
}
@dataclass(frozen=True)
class ParsedRow:
"""保留 Excel 行号的一行导入数据。"""
excel_row: int
values: dict[str, str]
class MasterExcelService:
"""基础资料 Excel 导入导出。七个资源共用列配置,一次导入只提交一次事务。"""
@classmethod
def build_template(cls, resource: MasterExcelResource) -> bytes:
"""生成只含业务列和状态下拉的导入模板。"""
spec = _SPECS[resource]
binary = ExcelUtil.get_excel_template(
header_list=list(spec.headers),
selector_header_list=['状态'],
option_list=[{'状态': ['正常', '停用']}],
)
return cls._format_text_columns(binary, len(spec.headers))
@classmethod
async def export_resource(
cls,
db: AsyncSession,
resource: MasterExcelResource,
query: BaseModel,
scope: TestBatchScope,
) -> bytes:
"""按当前筛选条件导出全部命中行,列与导入模板一致。"""
spec = _SPECS[resource]
records = await _LISTERS[resource](db, query, scope, is_page=False)
if not isinstance(records, list):
raise ServiceException(message='导出数据格式错误')
customer_codes: dict[int, str] = {}
site_codes: dict[int, str] = {}
if spec.parent_kind == 'customer':
customer_codes = await MasterDao.map_customer_codes(
db, {int(getattr(record, 'customer_id')) for record in records}
)
elif spec.parent_kind == 'site':
site_codes = await MasterDao.map_site_codes(db, {int(getattr(record, 'site_id')) for record in records})
data_rows = [
cls._export_row(spec, record, customer_codes, site_codes)
for record in records
]
return cls._fill_template(cls.build_template(resource), data_rows)
@classmethod
async def import_resource(
cls,
db: AsyncSession,
resource: MasterExcelResource,
file: UploadFile,
update_support: bool,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> CrudResponseModel:
"""导入一个资源。文件内编码重复或文件不可读时整批失败,单行业务错误只跳过该行。"""
parsed_rows = await cls._read_upload(resource, file)
if not parsed_rows:
return CrudResponseModel(is_success=True, message=cls._summary(0, 0, 0, []))
added = 0
updated = 0
skipped = 0
details: list[str] = []
try:
if resource == 'pallet':
await PalletService._prepare_test_write(db, current_user, scope)
else:
await MasterService._prepare_test_write(db, current_user, scope)
for row in parsed_rows:
try:
outcome, detail = await cls._apply_row(
db, resource, row, update_support, current_user, scope
)
except ServiceException as exc:
details.append(f'第{row.excel_row}行:{exc.message or "导入失败"}')
continue
if outcome == 'add':
added += 1
elif outcome == 'update':
updated += 1
else:
skipped += 1
if detail:
details.append(f'第{row.excel_row}行:{detail}')
if added or updated:
await db.commit()
else:
await db.rollback()
except (ServiceException, PermissionException):
await db.rollback()
raise
except Exception as exc:
await db.rollback()
raise ServiceException(message='导入失败,数据已全部回滚') from exc
return CrudResponseModel(is_success=True, message=cls._summary(added, updated, skipped, details))
@classmethod
async def _read_upload(cls, resource: MasterExcelResource, file: UploadFile) -> list[ParsedRow]:
spec = _SPECS[resource]
contents = await file.read()
await file.close()
if not contents:
raise ServiceException(message='导入文件不能为空')
if len(contents) > _MAX_IMPORT_BYTES:
raise ServiceException(message='导入文件不能超过5MB')
try:
headers, rows = ExcelUtil.read_text_table(contents)
except ValueError as exc:
raise ServiceException(message=str(exc)) from exc
if len(headers) != len(set(headers)) or set(headers) != set(spec.headers):
raise ServiceException(message=f'导入表头不匹配,应为:{"、".join(spec.headers)}')
if len(rows) > _MAX_IMPORT_ROWS:
raise ServiceException(message=f'一次最多导入{_MAX_IMPORT_ROWS}行')
parsed = [ParsedRow(excel_row=excel_row, values=values) for excel_row, values in rows]
cls._reject_duplicate_keys(spec, parsed)
return parsed
@staticmethod
def _reject_duplicate_keys(spec: ExcelSpec, rows: list[ParsedRow]) -> None:
seen: set[tuple[str, ...]] = set()
duplicates: list[str] = []
for row in rows:
identity = _identity(spec, row.values)
if identity is None or identity in seen:
if identity is not None:
duplicates.append(_identity_text(spec, row.values))
continue
seen.add(identity)
if duplicates:
unique_names = list(dict.fromkeys(duplicates))
raise ServiceException(message=f'导入文件存在重复编码:{"、".join(unique_names)}')
@classmethod
async def _apply_row(
cls,
db: AsyncSession,
resource: MasterExcelResource,
row: ParsedRow,
update_support: bool,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> tuple[ImportOutcome, str]:
spec = _SPECS[resource]
if resource == 'pallet':
code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max)
status = _status(row.values.get('状态', ''))
remark = _optional(row.values.get('备注', ''), '备注', 500)
outcome = await PalletService.stage_imported_pallet(
db,
pallet_code=code,
asset_status=status,
remark=remark,
update_support=update_support,
current_user=current_user,
scope=scope,
)
return outcome, f'托盘编号{code}已存在,已跳过' if outcome == 'skip' else ''
if resource in {'strain', 'customer', 'warehouse'}:
return await cls._apply_global_row(db, resource, row, update_support, current_user, scope)
if resource == 'site':
return await cls._apply_site_row(db, row, update_support, current_user, scope)
return await cls._apply_child_row(db, resource, row, update_support, current_user, scope)
@classmethod
async def _apply_global_row(
cls,
db: AsyncSession,
resource: Literal['strain', 'customer', 'warehouse'],
row: ParsedRow,
update_support: bool,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> tuple[ImportOutcome, str]:
spec = _SPECS[resource]
code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max)
name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max)
status = _status(row.values.get('状态', ''))
remark = _optional(row.values.get('备注', ''), '备注', 500)
address = (
_optional(row.values.get(spec.address_header, ''), spec.address_header or '地址', 500)
if spec.address_header
else None
)
found = await getattr(MasterDao, f'find_{resource}_by_code')(db, code)
if found is None:
await cls._save_master(
db,
resource,
existing=None,
parent_id=None,
code=code,
name=name,
status=status,
remark=remark,
address=address,
current_user=current_user,
scope=scope,
)
return 'add', ''
current = await getattr(MasterDao, f'get_{resource}')(
db,
getattr(found, spec.id_attr),
scope,
for_update=scope.mode == 'TEST',
)
if current is None:
raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围')
if not update_support:
return 'skip', f'{spec.code_header}{code}已存在,已跳过'
await cls._save_master(
db,
resource,
existing=current,
parent_id=None,
code=getattr(current, spec.code_attr),
name=name,
status=status,
remark=remark,
address=address,
current_user=current_user,
scope=scope,
)
return 'update', ''
@classmethod
async def _apply_site_row(
cls,
db: AsyncSession,
row: ParsedRow,
update_support: bool,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> tuple[ImportOutcome, str]:
spec = _SPECS['site']
customer_code = _required(row.values.get('客户编码', ''), '客户编码', 64)
site_code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max)
name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max)
status = _status(row.values.get('状态', ''))
remark = _optional(row.values.get('备注', ''), '备注', 500)
address = _optional(row.values.get('现场地址', ''), '现场地址', 500)
customer = await cls._require_customer(db, customer_code, scope)
found = await MasterDao.find_site_by_code(db, site_code, customer.customer_id)
if found is not None:
current = await MasterDao.get_site(db, found.site_id, scope, for_update=scope.mode == 'TEST')
if current is None:
raise ServiceException(message=f'{spec.code_header}{site_code}已存在于其他数据范围')
if not update_support:
return 'skip', f'客户{customer_code}的{spec.code_header}{site_code}已存在,已跳过'
await cls._save_master(
db,
'site',
existing=current,
parent_id=customer.customer_id,
code=current.site_code,
name=name,
status=status,
remark=remark,
address=address,
current_user=current_user,
scope=scope,
)
return 'update', ''
if update_support:
matches = await MasterDao.list_sites_by_code(db, site_code, scope)
if len(matches) > 1:
raise ServiceException(message=f'{spec.code_header}{site_code}不唯一,无法更新上级')
if len(matches) == 1:
current = await MasterDao.get_site(db, matches[0].site_id, scope, for_update=scope.mode == 'TEST')
if current is None:
raise ServiceException(message=f'{spec.code_header}{site_code}已存在于其他数据范围')
await cls._save_master(
db,
'site',
existing=current,
parent_id=customer.customer_id,
code=current.site_code,
name=name,
status=status,
remark=remark,
address=address,
current_user=current_user,
scope=scope,
)
return 'update', ''
await cls._save_master(
db,
'site',
existing=None,
parent_id=customer.customer_id,
code=site_code,
name=name,
status=status,
remark=remark,
address=address,
current_user=current_user,
scope=scope,
)
return 'add', ''
@classmethod
async def _apply_child_row(
cls,
db: AsyncSession,
resource: Literal['line', 'tunnel'],
row: ParsedRow,
update_support: bool,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> tuple[ImportOutcome, str]:
spec = _SPECS[resource]
site_code = _required(row.values.get('现场编码', ''), '现场编码', 64)
code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max)
name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max)
status = _status(row.values.get('状态', ''))
remark = _optional(row.values.get('备注', ''), '备注', 500)
site = await cls._require_site(db, site_code, scope)
found = await getattr(MasterDao, f'find_{resource}_by_code')(db, code, site.site_id)
if found is not None:
current = await getattr(MasterDao, f'get_{resource}')(
db, getattr(found, spec.id_attr), scope, for_update=scope.mode == 'TEST'
)
if current is None:
raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围')
if not update_support:
return 'skip', f'现场{site_code}的{spec.code_header}{code}已存在,已跳过'
await cls._save_master(
db,
resource,
existing=current,
parent_id=site.site_id,
code=getattr(current, spec.code_attr),
name=name,
status=status,
remark=remark,
address=None,
current_user=current_user,
scope=scope,
)
return 'update', ''
if update_support:
matches = await getattr(MasterDao, f'list_{resource}s_by_code')(db, code, scope)
if len(matches) > 1:
raise ServiceException(message=f'{spec.code_header}{code}不唯一,无法更新上级')
if len(matches) == 1:
current = await getattr(MasterDao, f'get_{resource}')(
db, getattr(matches[0], spec.id_attr), scope, for_update=scope.mode == 'TEST'
)
if current is None:
raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围')
await cls._save_master(
db,
resource,
existing=current,
parent_id=site.site_id,
code=getattr(current, spec.code_attr),
name=name,
status=status,
remark=remark,
address=None,
current_user=current_user,
scope=scope,
)
return 'update', ''
await cls._save_master(
db,
resource,
existing=None,
parent_id=site.site_id,
code=code,
name=name,
status=status,
remark=remark,
address=None,
current_user=current_user,
scope=scope,
)
return 'add', ''
@classmethod
async def _save_master(
cls,
db: AsyncSession,
resource: MasterResource,
*,
existing: object | None,
parent_id: int | None,
code: str,
name: str,
status: Literal['0', '1'],
remark: str | None,
address: str | None,
current_user: CurrentUserModel,
scope: TestBatchScope,
) -> None:
spec = _SPECS[resource]
now = datetime.now()
user_name = current_user.user.user_name
payload: dict[str, object] = {
spec.code_attr: code,
'status': status,
'remark': remark,
'update_by': user_name,
'update_time': now,
}
if spec.name_attr:
payload[spec.name_attr] = name
if spec.address_attr:
payload[spec.address_attr] = address
if spec.parent_attr:
payload[spec.parent_attr] = parent_id
if existing is None:
payload['create_by'] = user_name
payload['create_time'] = now
else:
payload[spec.id_attr] = getattr(existing, spec.id_attr)
try:
model = _MODEL_TYPES[resource](**payload)
except ValidationError as exc:
raise ServiceException(message=_validation_message(exc)) from exc
await MasterService._validate_parent(db, resource, model, scope)
current_id = None if existing is None else getattr(existing, spec.id_attr)
await MasterService._validate_unique(db, resource, model, current_id)
if existing is None:
await getattr(MasterDao, f'add_{resource}')(db, model, scope.batch_id)
return
await getattr(MasterDao, f'update_{resource}')(db, model, scope)
@staticmethod
async def _require_customer(db: AsyncSession, customer_code: str, scope: TestBatchScope):
found = await MasterDao.find_customer_by_code(db, customer_code)
if found is None:
raise ServiceException(message=f'目标客户{customer_code}不存在')
customer = await MasterDao.get_customer(db, found.customer_id, scope)
if customer is None:
raise ServiceException(message=f'目标客户{customer_code}不属于当前数据模式或测试批次')
if customer.status != '0':
raise ServiceException(message=f'目标客户{customer_code}已停用')
return customer
@staticmethod
async def _require_site(db: AsyncSession, site_code: str, scope: TestBatchScope):
matches = await MasterDao.list_sites_by_code(db, site_code, scope)
if not matches:
raise ServiceException(message=f'目标客户现场{site_code}不存在')
if len(matches) > 1:
raise ServiceException(message=f'现场编码{site_code}不唯一,无法确定目标客户现场')
site = matches[0]
if site.status != '0':
raise ServiceException(message=f'目标客户现场{site_code}已停用')
return site
@staticmethod
def _export_row(
spec: ExcelSpec,
record: BaseModel,
customer_codes: dict[int, str],
site_codes: dict[int, str],
) -> list[str]:
values: dict[str, str] = {}
if spec.parent_kind == 'customer':
customer_id = int(getattr(record, 'customer_id'))
customer_code = customer_codes.get(customer_id)
if not customer_code:
raise ServiceException(message=f'客户{customer_id}不存在,无法导出')
values[spec.parent_header or ''] = customer_code
elif spec.parent_kind == 'site':
site_id = int(getattr(record, 'site_id'))
site_code = site_codes.get(site_id)
if not site_code:
raise ServiceException(message=f'客户现场{site_id}不存在,无法导出')
values[spec.parent_header or ''] = site_code
values[spec.code_header] = str(getattr(record, spec.code_attr))
if spec.name_header and spec.name_attr:
values[spec.name_header] = str(getattr(record, spec.name_attr))
if spec.address_header and spec.address_attr:
values[spec.address_header] = str(getattr(record, spec.address_attr) or '')
status_value = str(getattr(record, spec.status_attr))
values['状态'] = _VALUE_TO_STATUS.get(status_value, status_value)
values['备注'] = str(getattr(record, 'remark') or '')
return [values.get(header, '') for header in spec.headers]
@staticmethod
def _summary(added: int, updated: int, skipped: int, details: list[str]) -> str:
summary = f'导入完成:新增{added}条,更新{updated}条,跳过{skipped}条'
if not details:
return summary
return summary + '\n' + '\n'.join(details)
@staticmethod
def _format_text_columns(binary: bytes, column_count: int) -> bytes:
workbook = load_workbook(io.BytesIO(binary))
worksheet = workbook.active
header_fill = PatternFill(start_color='ABABAB', end_color='ABABAB', fill_type='solid')
for column_index in range(1, column_count + 1):
column_letter = get_column_letter(column_index)
header = worksheet.cell(1, column_index)
header.fill = header_fill
header.alignment = Alignment(horizontal='center')
header.number_format = '@'
worksheet.column_dimensions[column_letter].number_format = '@'
buffer = io.BytesIO()
workbook.save(buffer)
return buffer.getvalue()
@staticmethod
def _fill_template(template: bytes, data_rows: list[list[str]]) -> bytes:
if not data_rows:
return template
workbook = load_workbook(io.BytesIO(template))
worksheet = workbook.active
for data in data_rows:
worksheet.append(data)
for column_index, _value in enumerate(data, start=1):
cell = worksheet.cell(worksheet.max_row, column_index)
if cell.value is not None:
cell.value = str(cell.value)
cell.number_format = '@'
buffer = io.BytesIO()
workbook.save(buffer)
return buffer.getvalue()
_MODEL_TYPES: dict[MasterResource, type[BaseModel]] = {
'strain': StrainModel,
'customer': CustomerModel,
'site': SiteModel,
'line': LineModel,
'warehouse': WarehouseModel,
'tunnel': TunnelModel,
}
def _required(value: str, label: str, max_length: int) -> str:
if not value:
raise ServiceException(message=f'{label}不能为空')
if len(value) > max_length:
raise ServiceException(message=f'{label}长度不能超过{max_length}')
return value
def _optional(value: str, label: str, max_length: int) -> str | None:
if not value:
return None
if len(value) > max_length:
raise ServiceException(message=f'{label}长度不能超过{max_length}')
return value
def _status(value: str) -> Literal['0', '1']:
mapped = _STATUS_TO_VALUE.get(value)
if mapped is None:
raise ServiceException(message='状态只能填写正常或停用')
return mapped
def _identity(spec: ExcelSpec, values: dict[str, str]) -> tuple[str, ...] | None:
code = values.get(spec.code_header, '')
if spec.parent_header:
parent = values.get(spec.parent_header, '')
if not code or not parent:
return None
return (parent.casefold(), code.casefold())
if not code:
return None
return (code.casefold(),)
def _identity_text(spec: ExcelSpec, values: dict[str, str]) -> str:
code = values.get(spec.code_header, '')
if spec.parent_header:
return f'{values.get(spec.parent_header, "")}/{code}'
return code
def _validation_message(exc: ValidationError) -> str:
error = exc.errors()[0]
location = error.get('loc') or ('字段',)
return f'{location[-1]}不合法'