import io from dataclasses import dataclass from datetime import datetime from typing import Literal from fastapi import UploadFile from openpyxl import load_workbook from openpyxl.styles import Alignment, PatternFill from openpyxl.utils import get_column_letter from pydantic import BaseModel, ValidationError from sqlalchemy.ext.asyncio import AsyncSession from common.vo import CrudResponseModel from exceptions.exception import PermissionException, ServiceException from module_admin.entity.vo.user_vo import CurrentUserModel from module_business.dao.master_dao import MasterDao from module_business.domain.test_batch_scope import TestBatchScope from module_business.entity.vo.master_summary_vo import MasterImportPreviewModel, MasterImportPreviewRowModel from module_business.entity.vo.master_vo import ( CustomerModel, LineModel, SiteModel, StrainModel, TunnelModel, WarehouseModel, ) from module_business.service.master_service import MasterResource, MasterService from module_business.service.pallet_service import PalletService from utils.excel_util import ExcelUtil MasterExcelResource = Literal['strain', 'customer', 'site', 'line', 'warehouse', 'tunnel', 'pallet'] ImportOutcome = Literal['add', 'update', 'skip'] _MAX_IMPORT_BYTES = 5 * 1024 * 1024 _MAX_IMPORT_ROWS = 5000 _STATUS_TO_VALUE = {'正常': '0', '停用': '1'} _VALUE_TO_STATUS = {'0': '正常', '1': '停用'} _PREVIEW_MESSAGES: dict[ImportOutcome, str] = { 'add': '校验通过,将新增', 'update': '{code}已存在,将按文件内容更新', 'skip': '{code}已存在,保持原资料不变', } @dataclass(frozen=True) class ExcelSpec: """一个基础资料资源的导入导出列定义。""" label: str headers: tuple[str, ...] code_header: str code_attr: str name_header: str | None = None name_attr: str | None = None address_header: str | None = None address_attr: str | None = None parent_header: str | None = None parent_kind: Literal['customer', 'site'] | None = None parent_attr: str | None = None status_attr: str = 'status' id_attr: str = '' code_max: int = 64 name_max: int = 100 _SPECS: dict[MasterExcelResource, ExcelSpec] = { 'strain': ExcelSpec( label='菌种', headers=('菌种编码', '菌种名称', '状态', '备注'), code_header='菌种编码', code_attr='strain_code', name_header='菌种名称', name_attr='strain_name', id_attr='strain_id', ), 'customer': ExcelSpec( label='客户', headers=('客户编码', '客户名称', '状态', '备注'), code_header='客户编码', code_attr='customer_code', name_header='客户名称', name_attr='customer_name', id_attr='customer_id', ), 'site': ExcelSpec( label='客户现场', headers=('客户编码', '现场编码', '现场名称', '现场地址', '状态', '备注'), code_header='现场编码', code_attr='site_code', name_header='现场名称', name_attr='site_name', address_header='现场地址', address_attr='address', parent_header='客户编码', parent_kind='customer', parent_attr='customer_id', id_attr='site_id', ), 'line': ExcelSpec( label='产线', headers=('现场编码', '产线编码', '产线名称', '状态', '备注'), code_header='产线编码', code_attr='line_code', name_header='产线名称', name_attr='line_name', parent_header='现场编码', parent_kind='site', parent_attr='site_id', id_attr='line_id', ), 'warehouse': ExcelSpec( label='仓库', headers=('仓库编码', '仓库名称', '仓库地址', '状态', '备注'), code_header='仓库编码', code_attr='warehouse_code', name_header='仓库名称', name_attr='warehouse_name', address_header='仓库地址', address_attr='address', id_attr='warehouse_id', ), 'pallet': ExcelSpec( label='托盘', headers=('托盘编号', '状态', '备注'), code_header='托盘编号', code_attr='pallet_code', status_attr='asset_status', id_attr='pallet_id', ), 'tunnel': ExcelSpec( label='隧道', headers=('现场编码', '隧道编号', '隧道名称', '状态', '备注'), code_header='隧道编号', code_attr='tunnel_code', name_header='隧道名称', name_attr='tunnel_name', parent_header='现场编码', parent_kind='site', parent_attr='site_id', id_attr='tunnel_id', ), } _LISTERS = { 'strain': MasterService.list_strain, 'customer': MasterService.list_customer, 'site': MasterService.list_site, 'line': MasterService.list_line, 'warehouse': MasterService.list_warehouse, 'tunnel': MasterService.list_tunnel, 'pallet': PalletService.list_pallet, } @dataclass(frozen=True) class ParsedRow: """保留 Excel 行号的一行导入数据。""" excel_row: int values: dict[str, str] class MasterExcelService: """基础资料 Excel 导入导出。七个资源共用列配置,一次导入只提交一次事务。""" @classmethod def build_template(cls, resource: MasterExcelResource) -> bytes: """生成只含业务列和状态下拉的导入模板。""" spec = _SPECS[resource] binary = ExcelUtil.get_excel_template( header_list=list(spec.headers), selector_header_list=['状态'], option_list=[{'状态': ['正常', '停用']}], ) return cls._format_text_columns(binary, len(spec.headers)) @classmethod async def export_resource( cls, db: AsyncSession, resource: MasterExcelResource, query: BaseModel, scope: TestBatchScope, ) -> bytes: """按当前筛选条件导出全部命中行,列与导入模板一致。""" spec = _SPECS[resource] records = await _LISTERS[resource](db, query, scope, is_page=False) if not isinstance(records, list): raise ServiceException(message='导出数据格式错误') customer_codes: dict[int, str] = {} site_codes: dict[int, str] = {} if spec.parent_kind == 'customer': customer_codes = await MasterDao.map_customer_codes( db, {int(getattr(record, 'customer_id')) for record in records} ) elif spec.parent_kind == 'site': site_codes = await MasterDao.map_site_codes(db, {int(getattr(record, 'site_id')) for record in records}) data_rows = [ cls._export_row(spec, record, customer_codes, site_codes) for record in records ] return cls._fill_template(cls.build_template(resource), data_rows) @classmethod async def import_resource( cls, db: AsyncSession, resource: MasterExcelResource, file: UploadFile, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> CrudResponseModel: """导入一个资源。文件内编码重复或文件不可读时整批失败,单行业务错误只跳过该行。""" parsed_rows = await cls._read_upload(resource, file) if not parsed_rows: return CrudResponseModel(is_success=True, message=cls._summary(0, 0, 0, [])) added = 0 updated = 0 skipped = 0 details: list[str] = [] try: if resource == 'pallet': await PalletService._prepare_test_write(db, current_user, scope) else: await MasterService._prepare_test_write(db, current_user, scope) for row in parsed_rows: try: outcome, detail = await cls._apply_row( db, resource, row, update_support, current_user, scope ) except ServiceException as exc: details.append(f'第{row.excel_row}行:{exc.message or "导入失败"}') continue if outcome == 'add': added += 1 elif outcome == 'update': updated += 1 else: skipped += 1 if detail: details.append(f'第{row.excel_row}行:{detail}') if added or updated: await db.commit() else: await db.rollback() except (ServiceException, PermissionException): await db.rollback() raise except Exception as exc: await db.rollback() raise ServiceException(message='导入失败,数据已全部回滚') from exc return CrudResponseModel(is_success=True, message=cls._summary(added, updated, skipped, details)) @classmethod async def preview_import( cls, db: AsyncSession, resource: MasterExcelResource, file: UploadFile, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> MasterImportPreviewModel: """走与正式导入完全相同的逐行校验,最后整体回滚,返回每行将发生的结果。""" spec = _SPECS[resource] parsed_rows = await cls._read_upload(resource, file) rows: list[MasterImportPreviewRowModel] = [] try: if resource == 'pallet': await PalletService._prepare_test_write(db, current_user, scope) else: await MasterService._prepare_test_write(db, current_user, scope) for row in parsed_rows: values = [row.values.get(header, '') for header in spec.headers] try: outcome, _ = await cls._apply_row(db, resource, row, update_support, current_user, scope) except ServiceException as exc: rows.append( MasterImportPreviewRowModel( excel_row=row.excel_row, outcome='error', message=exc.message or '校验失败', values=values, ) ) continue rows.append( MasterImportPreviewRowModel( excel_row=row.excel_row, outcome=outcome, message=_PREVIEW_MESSAGES[outcome].format(code=spec.code_header), values=values, ) ) except Exception as exc: if isinstance(exc, (ServiceException, PermissionException)): raise raise ServiceException(message='导入校验失败,请稍后重试') from exc finally: await db.rollback() counts = {outcome: sum(1 for row in rows if row.outcome == outcome) for outcome in _PREVIEW_MESSAGES} return MasterImportPreviewModel( headers=list(spec.headers), update_support=update_support, total_rows=len(rows), add_count=counts['add'], update_count=counts['update'], skip_count=counts['skip'], error_count=sum(1 for row in rows if row.outcome == 'error'), rows=rows, ) @classmethod async def _read_upload(cls, resource: MasterExcelResource, file: UploadFile) -> list[ParsedRow]: spec = _SPECS[resource] contents = await file.read() await file.close() if not contents: raise ServiceException(message='导入文件不能为空') if len(contents) > _MAX_IMPORT_BYTES: raise ServiceException(message='导入文件不能超过5MB') try: headers, rows = ExcelUtil.read_text_table(contents) except ValueError as exc: raise ServiceException(message=str(exc)) from exc if len(headers) != len(set(headers)) or set(headers) != set(spec.headers): raise ServiceException(message=f'导入表头不匹配,应为:{"、".join(spec.headers)}') if len(rows) > _MAX_IMPORT_ROWS: raise ServiceException(message=f'一次最多导入{_MAX_IMPORT_ROWS}行') parsed = [ParsedRow(excel_row=excel_row, values=values) for excel_row, values in rows] cls._reject_duplicate_keys(spec, parsed) return parsed @staticmethod def _reject_duplicate_keys(spec: ExcelSpec, rows: list[ParsedRow]) -> None: seen: set[tuple[str, ...]] = set() duplicates: list[str] = [] for row in rows: identity = _identity(spec, row.values) if identity is None or identity in seen: if identity is not None: duplicates.append(_identity_text(spec, row.values)) continue seen.add(identity) if duplicates: unique_names = list(dict.fromkeys(duplicates)) raise ServiceException(message=f'导入文件存在重复编码:{"、".join(unique_names)}') @classmethod async def _apply_row( cls, db: AsyncSession, resource: MasterExcelResource, row: ParsedRow, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> tuple[ImportOutcome, str]: spec = _SPECS[resource] if resource == 'pallet': code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max) status = _status(row.values.get('状态', '')) remark = _optional(row.values.get('备注', ''), '备注', 500) outcome = await PalletService.stage_imported_pallet( db, pallet_code=code, asset_status=status, remark=remark, update_support=update_support, current_user=current_user, scope=scope, ) return outcome, f'托盘编号{code}已存在,已跳过' if outcome == 'skip' else '' if resource in {'strain', 'customer', 'warehouse'}: return await cls._apply_global_row(db, resource, row, update_support, current_user, scope) if resource == 'site': return await cls._apply_site_row(db, row, update_support, current_user, scope) return await cls._apply_child_row(db, resource, row, update_support, current_user, scope) @classmethod async def _apply_global_row( cls, db: AsyncSession, resource: Literal['strain', 'customer', 'warehouse'], row: ParsedRow, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> tuple[ImportOutcome, str]: spec = _SPECS[resource] code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max) name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max) status = _status(row.values.get('状态', '')) remark = _optional(row.values.get('备注', ''), '备注', 500) address = ( _optional(row.values.get(spec.address_header, ''), spec.address_header or '地址', 500) if spec.address_header else None ) found = await getattr(MasterDao, f'find_{resource}_by_code')(db, code) if found is None: await cls._save_master( db, resource, existing=None, parent_id=None, code=code, name=name, status=status, remark=remark, address=address, current_user=current_user, scope=scope, ) return 'add', '' current = await getattr(MasterDao, f'get_{resource}')( db, getattr(found, spec.id_attr), scope, for_update=scope.mode == 'TEST', ) if current is None: raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围') if not update_support: return 'skip', f'{spec.code_header}{code}已存在,已跳过' await cls._save_master( db, resource, existing=current, parent_id=None, code=getattr(current, spec.code_attr), name=name, status=status, remark=remark, address=address, current_user=current_user, scope=scope, ) return 'update', '' @classmethod async def _apply_site_row( cls, db: AsyncSession, row: ParsedRow, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> tuple[ImportOutcome, str]: spec = _SPECS['site'] customer_code = _required(row.values.get('客户编码', ''), '客户编码', 64) site_code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max) name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max) status = _status(row.values.get('状态', '')) remark = _optional(row.values.get('备注', ''), '备注', 500) address = _optional(row.values.get('现场地址', ''), '现场地址', 500) customer = await cls._require_customer(db, customer_code, scope) found = await MasterDao.find_site_by_code(db, site_code, customer.customer_id) if found is not None: current = await MasterDao.get_site(db, found.site_id, scope, for_update=scope.mode == 'TEST') if current is None: raise ServiceException(message=f'{spec.code_header}{site_code}已存在于其他数据范围') if not update_support: return 'skip', f'客户{customer_code}的{spec.code_header}{site_code}已存在,已跳过' await cls._save_master( db, 'site', existing=current, parent_id=customer.customer_id, code=current.site_code, name=name, status=status, remark=remark, address=address, current_user=current_user, scope=scope, ) return 'update', '' if update_support: matches = await MasterDao.list_sites_by_code(db, site_code, scope) if len(matches) > 1: raise ServiceException(message=f'{spec.code_header}{site_code}不唯一,无法更新上级') if len(matches) == 1: current = await MasterDao.get_site(db, matches[0].site_id, scope, for_update=scope.mode == 'TEST') if current is None: raise ServiceException(message=f'{spec.code_header}{site_code}已存在于其他数据范围') await cls._save_master( db, 'site', existing=current, parent_id=customer.customer_id, code=current.site_code, name=name, status=status, remark=remark, address=address, current_user=current_user, scope=scope, ) return 'update', '' await cls._save_master( db, 'site', existing=None, parent_id=customer.customer_id, code=site_code, name=name, status=status, remark=remark, address=address, current_user=current_user, scope=scope, ) return 'add', '' @classmethod async def _apply_child_row( cls, db: AsyncSession, resource: Literal['line', 'tunnel'], row: ParsedRow, update_support: bool, current_user: CurrentUserModel, scope: TestBatchScope, ) -> tuple[ImportOutcome, str]: spec = _SPECS[resource] site_code = _required(row.values.get('现场编码', ''), '现场编码', 64) code = _required(row.values.get(spec.code_header, ''), spec.code_header, spec.code_max) name = _required(row.values.get(spec.name_header or '', ''), spec.name_header or '名称', spec.name_max) status = _status(row.values.get('状态', '')) remark = _optional(row.values.get('备注', ''), '备注', 500) site = await cls._require_site(db, site_code, scope) found = await getattr(MasterDao, f'find_{resource}_by_code')(db, code, site.site_id) if found is not None: current = await getattr(MasterDao, f'get_{resource}')( db, getattr(found, spec.id_attr), scope, for_update=scope.mode == 'TEST' ) if current is None: raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围') if not update_support: return 'skip', f'现场{site_code}的{spec.code_header}{code}已存在,已跳过' await cls._save_master( db, resource, existing=current, parent_id=site.site_id, code=getattr(current, spec.code_attr), name=name, status=status, remark=remark, address=None, current_user=current_user, scope=scope, ) return 'update', '' if update_support: matches = await getattr(MasterDao, f'list_{resource}s_by_code')(db, code, scope) if len(matches) > 1: raise ServiceException(message=f'{spec.code_header}{code}不唯一,无法更新上级') if len(matches) == 1: current = await getattr(MasterDao, f'get_{resource}')( db, getattr(matches[0], spec.id_attr), scope, for_update=scope.mode == 'TEST' ) if current is None: raise ServiceException(message=f'{spec.code_header}{code}已存在于其他数据范围') await cls._save_master( db, resource, existing=current, parent_id=site.site_id, code=getattr(current, spec.code_attr), name=name, status=status, remark=remark, address=None, current_user=current_user, scope=scope, ) return 'update', '' await cls._save_master( db, resource, existing=None, parent_id=site.site_id, code=code, name=name, status=status, remark=remark, address=None, current_user=current_user, scope=scope, ) return 'add', '' @classmethod async def _save_master( cls, db: AsyncSession, resource: MasterResource, *, existing: object | None, parent_id: int | None, code: str, name: str, status: Literal['0', '1'], remark: str | None, address: str | None, current_user: CurrentUserModel, scope: TestBatchScope, ) -> None: spec = _SPECS[resource] now = datetime.now() user_name = current_user.user.user_name payload: dict[str, object] = { spec.code_attr: code, 'status': status, 'remark': remark, 'update_by': user_name, 'update_time': now, } if spec.name_attr: payload[spec.name_attr] = name if spec.address_attr: payload[spec.address_attr] = address if spec.parent_attr: payload[spec.parent_attr] = parent_id if existing is None: payload['create_by'] = user_name payload['create_time'] = now else: payload[spec.id_attr] = getattr(existing, spec.id_attr) try: model = _MODEL_TYPES[resource](**payload) except ValidationError as exc: raise ServiceException(message=_validation_message(exc)) from exc await MasterService._validate_parent(db, resource, model, scope) current_id = None if existing is None else getattr(existing, spec.id_attr) await MasterService._validate_unique(db, resource, model, current_id) if existing is None: await getattr(MasterDao, f'add_{resource}')(db, model, scope.batch_id) return await getattr(MasterDao, f'update_{resource}')(db, model, scope) @staticmethod async def _require_customer(db: AsyncSession, customer_code: str, scope: TestBatchScope): found = await MasterDao.find_customer_by_code(db, customer_code) if found is None: raise ServiceException(message=f'目标客户{customer_code}不存在') customer = await MasterDao.get_customer(db, found.customer_id, scope) if customer is None: raise ServiceException(message=f'目标客户{customer_code}不属于当前数据模式或测试批次') if customer.status != '0': raise ServiceException(message=f'目标客户{customer_code}已停用') return customer @staticmethod async def _require_site(db: AsyncSession, site_code: str, scope: TestBatchScope): matches = await MasterDao.list_sites_by_code(db, site_code, scope) if not matches: raise ServiceException(message=f'目标客户现场{site_code}不存在') if len(matches) > 1: raise ServiceException(message=f'现场编码{site_code}不唯一,无法确定目标客户现场') site = matches[0] if site.status != '0': raise ServiceException(message=f'目标客户现场{site_code}已停用') return site @staticmethod def _export_row( spec: ExcelSpec, record: BaseModel, customer_codes: dict[int, str], site_codes: dict[int, str], ) -> list[str]: values: dict[str, str] = {} if spec.parent_kind == 'customer': customer_id = int(getattr(record, 'customer_id')) customer_code = customer_codes.get(customer_id) if not customer_code: raise ServiceException(message=f'客户{customer_id}不存在,无法导出') values[spec.parent_header or ''] = customer_code elif spec.parent_kind == 'site': site_id = int(getattr(record, 'site_id')) site_code = site_codes.get(site_id) if not site_code: raise ServiceException(message=f'客户现场{site_id}不存在,无法导出') values[spec.parent_header or ''] = site_code values[spec.code_header] = str(getattr(record, spec.code_attr)) if spec.name_header and spec.name_attr: values[spec.name_header] = str(getattr(record, spec.name_attr)) if spec.address_header and spec.address_attr: values[spec.address_header] = str(getattr(record, spec.address_attr) or '') status_value = str(getattr(record, spec.status_attr)) values['状态'] = _VALUE_TO_STATUS.get(status_value, status_value) values['备注'] = str(getattr(record, 'remark') or '') return [values.get(header, '') for header in spec.headers] @staticmethod def _summary(added: int, updated: int, skipped: int, details: list[str]) -> str: summary = f'导入完成:新增{added}条,更新{updated}条,跳过{skipped}条' if not details: return summary return summary + '\n' + '\n'.join(details) @staticmethod def _format_text_columns(binary: bytes, column_count: int) -> bytes: workbook = load_workbook(io.BytesIO(binary)) worksheet = workbook.active header_fill = PatternFill(start_color='ABABAB', end_color='ABABAB', fill_type='solid') for column_index in range(1, column_count + 1): column_letter = get_column_letter(column_index) header = worksheet.cell(1, column_index) header.fill = header_fill header.alignment = Alignment(horizontal='center') header.number_format = '@' worksheet.column_dimensions[column_letter].number_format = '@' buffer = io.BytesIO() workbook.save(buffer) return buffer.getvalue() @staticmethod def _fill_template(template: bytes, data_rows: list[list[str]]) -> bytes: if not data_rows: return template workbook = load_workbook(io.BytesIO(template)) worksheet = workbook.active for data in data_rows: worksheet.append(data) for column_index, _value in enumerate(data, start=1): cell = worksheet.cell(worksheet.max_row, column_index) if cell.value is not None: cell.value = str(cell.value) cell.number_format = '@' buffer = io.BytesIO() workbook.save(buffer) return buffer.getvalue() _MODEL_TYPES: dict[MasterResource, type[BaseModel]] = { 'strain': StrainModel, 'customer': CustomerModel, 'site': SiteModel, 'line': LineModel, 'warehouse': WarehouseModel, 'tunnel': TunnelModel, } def _required(value: str, label: str, max_length: int) -> str: if not value: raise ServiceException(message=f'{label}不能为空') if len(value) > max_length: raise ServiceException(message=f'{label}长度不能超过{max_length}') return value def _optional(value: str, label: str, max_length: int) -> str | None: if not value: return None if len(value) > max_length: raise ServiceException(message=f'{label}长度不能超过{max_length}') return value def _status(value: str) -> Literal['0', '1']: mapped = _STATUS_TO_VALUE.get(value) if mapped is None: raise ServiceException(message='状态只能填写正常或停用') return mapped def _identity(spec: ExcelSpec, values: dict[str, str]) -> tuple[str, ...] | None: code = values.get(spec.code_header, '') if spec.parent_header: parent = values.get(spec.parent_header, '') if not code or not parent: return None return (parent.casefold(), code.casefold()) if not code: return None return (code.casefold(),) def _identity_text(spec: ExcelSpec, values: dict[str, str]) -> str: code = values.get(spec.code_header, '') if spec.parent_header: return f'{values.get(spec.parent_header, "")}/{code}' return code def _validation_message(exc: ValidationError) -> str: error = exc.errors()[0] location = error.get('loc') or ('字段',) return f'{location[-1]}不合法'