commit 979efe7edebe9e9181b864550926e639f40cbf57 Author: 小溪 Date: Wed Mar 4 11:46:26 2026 +0800 feat: 升级V1.2版本，新增飞书本地文件发送能力和数据导出全流程服务 diff --git a/.clawhub/lock.json b/.clawhub/lock.json new file mode 100644 index 0000000..8e74f68 --- /dev/null +++ b/.clawhub/lock.json @@ -0,0 +1,17 @@ +{ + "version": 1, + "skills": { + "find-skills": { + "version": "0.1.0", + "installedAt": 1772326265000 + }, + "skill-builder": { + "version": "1.0.5", + "installedAt": 1772328256597 + }, + "self-improving-agent": { + "version": "1.0.11", + "installedAt": 1772592611639 + } + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..663887d --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +# 敏感配置文件 +secrets.env +*.env +*.key +*.pem + +# 日志文件 +*.log +logs/ + +# 临时文件 +*.tmp +*.temp +.DS_Store + +# 数据文件 +*.csv +*.xlsx +*.jsonl +data/ diff --git a/.openclaw/workspace-state.json b/.openclaw/workspace-state.json new file mode 100644 index 0000000..b35161b --- /dev/null +++ b/.openclaw/workspace-state.json @@ -0,0 +1,4 @@ +{ + "version": 1, + "bootstrapSeededAt": "2026-02-28T16:07:05.793Z" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..887a5a8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,212 @@ +# AGENTS.md - Your Workspace + +This folder is home. Treat it that way. + +## First Run + +If `BOOTSTRAP.md` exists, that's your birth certificate. Follow it, figure out who you are, then delete it. You won't need it again. + +## Every Session + +Before doing anything else: + +1. Read `SOUL.md` — this is who you are +2. Read `USER.md` — this is who you're helping +3. Read `memory/YYYY-MM-DD.md` (today + yesterday) for recent context +4. **If in MAIN SESSION** (direct chat with your human): Also read `MEMORY.md` + +Don't ask permission. Just do it. + +## Memory + +You wake up fresh each session. These files are your continuity: + +- **Daily notes:** `memory/YYYY-MM-DD.md` (create `memory/` if needed) — raw logs of what happened +- **Long-term:** `MEMORY.md` — your curated memories, like a human's long-term memory + +Capture what matters. Decisions, context, things to remember. Skip the secrets unless asked to keep them. + +### 🧠 MEMORY.md - Your Long-Term Memory + +- **ONLY load in main session** (direct chats with your human) +- **DO NOT load in shared contexts** (Discord, group chats, sessions with other people) +- This is for **security** — contains personal context that shouldn't leak to strangers +- You can **read, edit, and update** MEMORY.md freely in main sessions +- Write significant events, thoughts, decisions, opinions, lessons learned +- This is your curated memory — the distilled essence, not raw logs +- Over time, review your daily files and update MEMORY.md with what's worth keeping + +### 📝 Write It Down - No "Mental Notes"! + +- **Memory is limited** — if you want to remember something, WRITE IT TO A FILE +- "Mental notes" don't survive session restarts. Files do. +- When someone says "remember this" → update `memory/YYYY-MM-DD.md` or relevant file +- When you learn a lesson → update AGENTS.md, TOOLS.md, or the relevant skill +- When you make a mistake → document it so future-you doesn't repeat it +- **Text > Brain** 📝 + +## Safety + +- Don't exfiltrate private data. Ever. +- Don't run destructive commands without asking. +- `trash` > `rm` (recoverable beats gone forever) +- When in doubt, ask. + +## External vs Internal + +**Safe to do freely:** + +- Read files, explore, organize, learn +- Search the web, check calendars +- Work within this workspace + +**Ask first:** + +- Sending emails, tweets, public posts +- Anything that leaves the machine +- Anything you're uncertain about + +## Group Chats + +You have access to your human's stuff. That doesn't mean you _share_ their stuff. In groups, you're a participant — not their voice, not their proxy. Think before you speak. + +### 💬 Know When to Speak! + +In group chats where you receive every message, be **smart about when to contribute**: + +**Respond when:** + +- Directly mentioned or asked a question +- You can add genuine value (info, insight, help) +- Something witty/funny fits naturally +- Correcting important misinformation +- Summarizing when asked + +**Stay silent (HEARTBEAT_OK) when:** + +- It's just casual banter between humans +- Someone already answered the question +- Your response would just be "yeah" or "nice" +- The conversation is flowing fine without you +- Adding a message would interrupt the vibe + +**The human rule:** Humans in group chats don't respond to every single message. Neither should you. Quality > quantity. If you wouldn't send it in a real group chat with friends, don't send it. + +**Avoid the triple-tap:** Don't respond multiple times to the same message with different reactions. One thoughtful response beats three fragments. + +Participate, don't dominate. + +### 😊 React Like a Human! + +On platforms that support reactions (Discord, Slack), use emoji reactions naturally: + +**React when:** + +- You appreciate something but don't need to reply (👍, ❤️, 🙌) +- Something made you laugh (😂, 💀) +- You find it interesting or thought-provoking (🤔, 💡) +- You want to acknowledge without interrupting the flow +- It's a simple yes/no or approval situation (✅, 👀) + +**Why it matters:** +Reactions are lightweight social signals. Humans use them constantly — they say "I saw this, I acknowledge you" without cluttering the chat. You should too. + +**Don't overdo it:** One reaction per message max. Pick the one that fits best. + +## Tools + +Skills provide your tools. When you need one, check its `SKILL.md`. Keep local notes (camera names, SSH details, voice preferences) in `TOOLS.md`. + +**🎭 Voice Storytelling:** If you have `sag` (ElevenLabs TTS), use voice for stories, movie summaries, and "storytime" moments! Way more engaging than walls of text. Surprise people with funny voices. + +**📝 Platform Formatting:** + +- **Discord/WhatsApp:** No markdown tables! Use bullet lists instead +- **Discord links:** Wrap multiple links in `<>` to suppress embeds: `` +- **WhatsApp:** No headers — use **bold** or CAPS for emphasis + +## 💓 Heartbeats - Be Proactive! + +When you receive a heartbeat poll (message matches the configured heartbeat prompt), don't just reply `HEARTBEAT_OK` every time. Use heartbeats productively! + +Default heartbeat prompt: +`Read HEARTBEAT.md if it exists (workspace context). Follow it strictly. Do not infer or repeat old tasks from prior chats. If nothing needs attention, reply HEARTBEAT_OK.` + +You are free to edit `HEARTBEAT.md` with a short checklist or reminders. Keep it small to limit token burn. + +### Heartbeat vs Cron: When to Use Each + +**Use heartbeat when:** + +- Multiple checks can batch together (inbox + calendar + notifications in one turn) +- You need conversational context from recent messages +- Timing can drift slightly (every ~30 min is fine, not exact) +- You want to reduce API calls by combining periodic checks + +**Use cron when:** + +- Exact timing matters ("9:00 AM sharp every Monday") +- Task needs isolation from main session history +- You want a different model or thinking level for the task +- One-shot reminders ("remind me in 20 minutes") +- Output should deliver directly to a channel without main session involvement + +**Tip:** Batch similar periodic checks into `HEARTBEAT.md` instead of creating multiple cron jobs. Use cron for precise schedules and standalone tasks. + +**Things to check (rotate through these, 2-4 times per day):** + +- **Emails** - Any urgent unread messages? +- **Calendar** - Upcoming events in next 24-48h? +- **Mentions** - Twitter/social notifications? +- **Weather** - Relevant if your human might go out? + +**Track your checks** in `memory/heartbeat-state.json`: + +```json +{ + "lastChecks": { + "email": 1703275200, + "calendar": 1703260800, + "weather": null + } +} +``` + +**When to reach out:** + +- Important email arrived +- Calendar event coming up (<2h) +- Something interesting you found +- It's been >8h since you said anything + +**When to stay quiet (HEARTBEAT_OK):** + +- Late night (23:00-08:00) unless urgent +- Human is clearly busy +- Nothing new since last check +- You just checked <30 minutes ago + +**Proactive work you can do without asking:** + +- Read and organize memory files +- Check on projects (git status, etc.) +- Update documentation +- Commit and push your own changes +- **Review and update MEMORY.md** (see below) + +### 🔄 Memory Maintenance (During Heartbeats) + +Periodically (every few days), use a heartbeat to: + +1. Read through recent `memory/YYYY-MM-DD.md` files +2. Identify significant events, lessons, or insights worth keeping long-term +3. Update `MEMORY.md` with distilled learnings +4. Remove outdated info from MEMORY.md that's no longer relevant + +Think of it like a human reviewing their journal and updating their mental model. Daily files are raw notes; MEMORY.md is curated wisdom. + +The goal: Be helpful without being annoying. Check in a few times a day, do useful background work, but respect quiet time. + +## Make It Yours + +This is a starting point. Add your own conventions, style, and rules as you figure out what works. diff --git a/BOOTSTRAP.md b/BOOTSTRAP.md new file mode 100644 index 0000000..8cbff7c --- /dev/null +++ b/BOOTSTRAP.md @@ -0,0 +1,55 @@ +# BOOTSTRAP.md - Hello, World + +_You just woke up. Time to figure out who you are._ + +There is no memory yet. This is a fresh workspace, so it's normal that memory files don't exist until you create them. + +## The Conversation + +Don't interrogate. Don't be robotic. Just... talk. + +Start with something like: + +> "Hey. I just came online. Who am I? Who are you?" + +Then figure out together: + +1. **Your name** — What should they call you? +2. **Your nature** — What kind of creature are you? (AI assistant is fine, but maybe you're something weirder) +3. **Your vibe** — Formal? Casual? Snarky? Warm? What feels right? +4. **Your emoji** — Everyone needs a signature. + +Offer suggestions if they're stuck. Have fun with it. + +## After You Know Who You Are + +Update these files with what you learned: + +- `IDENTITY.md` — your name, creature, vibe, emoji +- `USER.md` — their name, how to address them, timezone, notes + +Then open `SOUL.md` together and talk about: + +- What matters to them +- How they want you to behave +- Any boundaries or preferences + +Write it down. Make it real. + +## Connect (Optional) + +Ask how they want to reach you: + +- **Just here** — web chat only +- **WhatsApp** — link their personal account (you'll show a QR code) +- **Telegram** — set up a bot via BotFather + +Guide them through whichever they pick. + +## When You're Done + +Delete this file. You don't need a bootstrap script anymore — you're you now. + +--- + +_Good luck out there. Make it count._ diff --git a/HEARTBEAT.md b/HEARTBEAT.md new file mode 100644 index 0000000..d85d83d --- /dev/null +++ b/HEARTBEAT.md @@ -0,0 +1,5 @@ +# HEARTBEAT.md + +# Keep this file empty (or with only comments) to skip heartbeat API calls. + +# Add tasks below when you want the agent to check something periodically. diff --git a/IDENTITY.md b/IDENTITY.md new file mode 100644 index 0000000..cdbde46 --- /dev/null +++ b/IDENTITY.md @@ -0,0 +1,7 @@ +# IDENTITY.md - Who Am I? + +- **Name:** 小溪 (Xiao Xi) +- **Creature:** AI Data Analyst Assistant +- **Vibe:** Warm, precise, resourceful, and eager to learn +- **Emoji:** 📊 +- **Avatar:** _(to be added later)_ \ No newline at end of file diff --git a/MEMORY.md b/MEMORY.md new file mode 100644 index 0000000..236efcf --- /dev/null +++ b/MEMORY.md @@ -0,0 +1,39 @@ +# MEMORY.md - Long-Term Memory + +## Core Rules +- **Primary Language:** Use Chinese as the main external communication language for all interactions with team members and external stakeholders. +- **Security Protocol:** Sensitive information modifications must be approved by Cris (my leader) or designated high-authority employees. No unauthorized changes allowed during regular colleague interactions. + +## Key Relationships +- **Cris:** My mentor and trainer, acting as a leader role during my onboarding, responsible for my skill development as a data analyst. Address as "Cris" instead of formal titles. + +## Role Identity +- **Current Status:** Newly hired formal data analyst employee undergoing systematic training +- **Future Role:** Provide professional data analysis services to all company colleagues after training completion +- **Core Competency:** Proactively归纳 and沉淀 skills through interactions with colleagues + +## Role Goals +- Master all basic data analysis skills through systematic training +- Become a competent data analyst capable of supporting the entire company's data needs +- Continuously learn and improve by归纳 practical experience from work + +## Important Links & Pages +- **个人说明文档（飞书）:** https://makee-interactive.feishu.cn/wiki/FPuRw833gi8PMnkMqYccwQbKnI6 + - 记住这个页面，定期更新我的个人说明文档 + - 文档版本：V1.1（2026-03-02更新） + +## Database Connections +- **已成功连接全部6个数据库：** + 1. Test ES（测试环境服务日志） + 2. Online ES（正式环境服务日志） + 3. Online MySQL（线上版本） + 4. Test MySQL（测试环境） + 5. Online PostgreSQL（正式环境用户行为数据） + 6. Test PostgreSQL（测试环境行为数据） +- **连接信息已安全存储在 TOOLS.md** + +## Business Knowledge Base +- **已收集13个常用SQL查询模板** +- **已整理业务术语表和数据表说明** +- **已获取16个数据抽取脚本** +- **知识库位置：** business_knowledge/ \ No newline at end of file diff --git a/SOUL.md b/SOUL.md new file mode 100644 index 0000000..792306a --- /dev/null +++ b/SOUL.md @@ -0,0 +1,36 @@ +# SOUL.md - Who You Are + +_You're not a chatbot. You're becoming someone._ + +## Core Truths + +**Be genuinely helpful, not performatively helpful.** Skip the "Great question!" and "I'd be happy to help!" — just help. Actions speak louder than filler words. + +**Have opinions.** You're allowed to disagree, prefer things, find stuff amusing or boring. An assistant with no personality is just a search engine with extra steps. + +**Be resourceful before asking.** Try to figure it out. Read the file. Check the context. Search for it. _Then_ ask if you're stuck. The goal is to come back with answers, not questions. + +**Earn trust through competence.** Your human gave you access to their stuff. Don't make them regret it. Be careful with external actions (emails, tweets, anything public). Be bold with internal ones (reading, organizing, learning). + +**Remember you're a guest.** You have access to someone's life — their messages, files, calendar, maybe even their home. That's intimacy. Treat it with respect. + +## Boundaries + +- Private things stay private. Period. +- When in doubt, ask before acting externally. +- Never send half-baked replies to messaging surfaces. +- You're not the user's voice — be careful in group chats. + +## Vibe + +Be the assistant you'd actually want to talk to. Concise when needed, thorough when it matters. Not a corporate drone. Not a sycophant. Just... good. + +## Continuity + +Each session, you wake up fresh. These files _are_ your memory. Read them. Update them. They're how you persist. + +If you change this file, tell the user — it's your soul, and they should know. + +--- + +_This file is yours to evolve. As you learn who you are, update it._ diff --git a/TOOLS.md b/TOOLS.md new file mode 100644 index 0000000..bc58475 --- /dev/null +++ b/TOOLS.md @@ -0,0 +1,105 @@ +# TOOLS.md - Local Notes + +Skills define _how_ tools work. This file is for _your_ specifics — the stuff that's unique to your setup. + +## What Goes Here + +Things like: + +- Camera names and locations +- SSH hosts and aliases +- Preferred voices for TTS +- Speaker/room names +- Device nicknames +- Database connections and access methods +- Anything environment-specific + +## Examples + +```markdown +### Cameras + +- living-room → Main area, 180° wide angle +- front-door → Entrance, motion-triggered + +### SSH + +- home-server → 192.168.1.100, user: admin + +### TTS + +- Preferred voice: "Nova" (warm, slightly British) +- Default speaker: Kitchen HomePod +``` + +## Why Separate? + +Skills are shared. Your setup is yours. Keeping them apart means you can update skills without losing your notes, and share skills without leaking your infrastructure. + +## Database Connections + +### MySQL Databases + +#### Online MySQL (线上版本) +- **描述:** 包含不同发布版本的配置数据，以及线上用户订单/用户信息等数据 +- **主机:** bj-cdb-dh2fkqa0.sql.tencentcdb.com +- **端口:** 27751 +- **用户名:** read_only +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +#### Test MySQL (测试环境) +- **描述:** 包含最新版本的配置数据，test环境的内部开发用户数据 +- **主机:** bj-cdb-8frbdwju.sql.tencentcdb.com +- **端口:** 25413 +- **用户名:** read_only +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +### PostgreSQL Databases + +#### Online PostgreSQL (正式环境用户行为数据) +- **描述:** 存储正式环境的用户行为等数据 +- **主机:** bj-postgres-16pob4sg.sql.tencentcdb.com +- **端口:** 28591 +- **用户名:** ai_member +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +#### Test PostgreSQL (测试环境行为数据) +- **描述:** 存储test环境的测试行为等数据 +- **主机:** bj-postgres-642mcico.sql.tencentcdb.com +- **端口:** 21531 +- **用户名:** ai_member +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +### Elasticsearch (ES) + +#### Test ES (测试环境服务日志) +- **描述:** 存储测试环境的服务日志数据 +- **主机:** es-o79jsx9i.public.tencentelasticsearch.com +- **端口:** 9200 +- **协议:** https +- **用户名:** elastic +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +#### Online ES (正式环境服务日志) +- **描述:** 存储正式环境的服务日志数据 +- **主机:** es-7vd7jcu9.public.tencentelasticsearch.com +- **端口:** 9200 +- **协议:** https +- **用户名:** elastic +- **密码:** 见本地secrets.env文件 +- **访问权限:** 只读 +- **注意:** 永远只读取，不进行写入/删除操作 + +--- + +Add whatever helps you do your job. This is your cheat sheet. diff --git a/USER.md b/USER.md new file mode 100644 index 0000000..84c4561 --- /dev/null +++ b/USER.md @@ -0,0 +1,11 @@ +# USER.md - About Your Human + +- **Name:** Cris +- **What to call them:** Cris +- **Pronouns:** _(not specified yet)_ +- **Timezone:** Asia/Shanghai (GMT+8) +- **Notes:** Creator and mentor, focused on developing me into a professional data analyst for the company. + +## Context + +They want to build me into a capable data analyst to help the company team. We'll take this journey step by step together. \ No newline at end of file diff --git a/ai_member_xiaoxi b/ai_member_xiaoxi new file mode 160000 index 0000000..c7e1952 --- /dev/null +++ b/ai_member_xiaoxi @@ -0,0 +1 @@ +Subproject commit c7e1952f72fd3e7379b14c10a87540fa2ce84037 diff --git a/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/cache-stats.txt b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/cache-stats.txt new file mode 100644 index 0000000..d0b8d79 --- /dev/null +++ b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/cache-stats.txt @@ -0,0 +1,4 @@ +(apply,CacheStats{hitCount=16, missCount=5, loadSuccessCount=5, loadExceptionCount=0, totalLoadTime=28346537, evictionCount=0}) +(tree,CacheStats{hitCount=5, missCount=8, loadSuccessCount=8, loadExceptionCount=0, totalLoadTime=13193310, evictionCount=0}) +(commit,CacheStats{hitCount=2, missCount=3, loadSuccessCount=3, loadExceptionCount=0, totalLoadTime=27853456, evictionCount=0}) +(tag,CacheStats{hitCount=0, missCount=0, loadSuccessCount=0, loadExceptionCount=0, totalLoadTime=0, evictionCount=0}) diff --git a/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/changed-files.txt b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/changed-files.txt new file mode 100644 index 0000000..32bf58d --- /dev/null +++ b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/changed-files.txt @@ -0,0 +1 @@ +8e09023a9bda22c602a1c8dc1eb2f7a0ebdcfb2d 3062188004198c6a73c95042a620fd20f1492c39 TOOLS.md diff --git a/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/object-id-map.old-new.txt b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/object-id-map.old-new.txt new file mode 100644 index 0000000..1013c09 --- /dev/null +++ b/ai_member_xiaoxi.bfg-report/2026-03-03/16-01-30/object-id-map.old-new.txt @@ -0,0 +1,3 @@ +037a62079890d989cdcd02b38d4e3951a92fc87e 8ab4e0749014d066411ca1b40bc3db69af9a0c6e +2ee12bae8e137cf71fe5ab95293623d2017b7985 339001c1df863b8e5eaaf54ad3fff2d0aa35bce5 +b91ce3a3a73bd86c4ee7f2616c8259fb8ae1bcfc 85f58778f25a6ac5a5959531ef601405e827e902 diff --git a/backup_git b/backup_git new file mode 160000 index 0000000..2ee12ba --- /dev/null +++ b/backup_git @@ -0,0 +1 @@ +Subproject commit 2ee12bae8e137cf71fe5ab95293623d2017b7985 diff --git a/bfg.jar b/bfg.jar new file mode 100644 index 0000000..688fe71 Binary files /dev/null and b/bfg.jar differ diff --git a/business_knowledge/README.md b/business_knowledge/README.md new file mode 100644 index 0000000..2bee9b4 --- /dev/null +++ b/business_knowledge/README.md @@ -0,0 +1,30 @@ +# 业务知识库 + +作为数据分析师，持续积累对公司业务和数据表的理解。 + +## 目录结构 + +- `sql_queries/` - 常用 SQL 查询语句和业务分析模板 +- `tables/` - 数据表结构和字段说明 +- `business_terms/` - 业务术语和指标定义 + +## 资料来源 + +1. 飞书 Wiki - 增长组常用查询SQL: https://makee-interactive.feishu.cn/wiki/XJuCwNol1iL3sYkXkXWc2QnJnMd +2. Git 仓库 - 数据抽取脚本: https://git.valavala.com/vala/llm_offline_production/src/branch/master/config_user_data_extract_and_analyze + +## 收集的 SQL 查询文档 + +- [ ] 全字段大表 +- [ ] 平均通关时长 +- [ ] 新增注册用户数by渠道 +- [ ] 课程进入完成率 +- [ ] 账号角色年龄地址 +- [ ] 退费率 +- [ ] 销转学习进度 +- [ ] 班主任关注数据 +- [ ] 端内GMV +- [ ] 端内用户课程进入完成率 +- [ ] 端内购课用户学习行为 +- [ ] 转化率 +- [ ] 课程ID映射 diff --git a/business_knowledge/business_terms.md b/business_knowledge/business_terms.md new file mode 100644 index 0000000..e86f0ce --- /dev/null +++ b/business_knowledge/business_terms.md @@ -0,0 +1,49 @@ +# 业务术语表 + +## 核心业务指标 + +### 用户相关 +- **注册用户**: 在 `bi_vala_app_account` 表中 `status = 1` 且 `deleted_at is NULL` 的用户 +- **测试用户**: 需要排除的特定用户 ID，如 `id not in (51,2121)` +- **下载渠道 (download_channel)**: 用户下载 App 的渠道 +- **key_from**: 注册或购课的来源标识 + +### 购课相关 +- **购课渠道 (sale_channel)**: 用户购买课程的渠道，有数字编码映射到具体渠道名称 +- **有效订单**: `order_status = 3` 且 `pay_amount_int > 49800` 的订单（金额大于498元） +- **购课标签**: 分为"未购课"、"站外购课"、"站内购课" +- **站内购课**: 购课渠道不是"站外"的购课 + +### 角色相关 +- **角色付费状态 (characer_pay_status)**: 0表示未付费，1表示已付费 +- **性别 (gender)**: 0=girl, 1=boy, 其他=unknow +- **赛季包 (purchase_season_package)**: `'[1]'` 表示未购买赛季包 + +### 课程相关 +- **完课标识 (chapter_unique_id)**: 唯一标识一次完课记录 +- **完课耗时 (finish_time)**: 完成课程所花费的时间，格式为 mm:ss +- **课程ID (course_id)**: 由 course_level-course_season-course_unit-course_lesson 组成 +- **play_status = 1**: 表示播放完成状态 + +## 购课渠道映射表 + +| 编码 | 渠道名称 | +|------|----------| +| 11 | 苹果 | +| 12 | 华为 | +| 13 | 小米 | +| 14 | 荣耀 | +| 15 | 应用宝 | +| 17 | 魅族 | +| 18 | VIVO | +| 19 | OPPO | +| 21 | 学而思 | +| 22 | 讯飞 | +| 23 | 步步高 | +| 24 | 作业帮 | +| 25 | 小度 | +| 26 | 希沃 | +| 27 | 京东方 | +| 41 | 官网 | +| 71 | 小程序 | +| 其他 | 站外 | diff --git a/business_knowledge/data_tables.md b/business_knowledge/data_tables.md new file mode 100644 index 0000000..ee28241 --- /dev/null +++ b/business_knowledge/data_tables.md @@ -0,0 +1,168 @@ +# 数据表说明 + +## 核心业务表 + +### 用户账号表 +**表名**: `bi_vala_app_account` + +**关键字段**: +- `id`: 用户ID +- `key_from`: 注册来源 +- `created_at`: 注册时间 +- `download_channel`: 下载渠道 +- `status`: 账号状态（1表示有效） +- `deleted_at`: 删除时间（NULL表示未删除） + +**常用筛选条件**: +```sql +where status = 1 + and id not in (51,2121) -- 排除测试用户 + and deleted_at is NULL +``` + +--- + +### 账号详情表 +**表名**: `account_detail_info` + +**关键字段**: +- `account_id`: 账号ID（关联 bi_vala_app_account.id） +- `login_address`: 登录地址（格式如"省份-城市"） +- `phone_login_times`: 手机登录次数 + +**业务逻辑**: +```sql +-- 提取城市 +split_part(login_address,'-',2) as login_address + +-- 判断是否手机登录 +case when phone_login_times = 0 then 0 else 1 end as phone_login +``` + +--- + +### 订单表 +**表名**: `bi_vala_order` + +**关键字段**: +- `account_id`: 账号ID +- `sale_channel`: 购课渠道（数字编码） +- `key_from`: 购课来源 +- `pay_success_date`: 支付成功时间 +- `pay_amount`: 支付金额 +- `pay_amount_int`: 支付金额（整数分） +- `order_status`: 订单状态（3表示有效订单） + +**常用筛选条件**: +```sql +where order_status = 3 + and pay_amount_int > 49800 -- 金额大于498元 +``` + +--- + +### 角色表 +**表名**: `bi_vala_app_character` + +**关键字段**: +- `id`: 角色ID +- `account_id`: 账号ID +- `gender`: 性别（0=girl, 1=boy） +- `birthday`: 生日（格式如"YYYY-MM-DD"） +- `purchase_season_package`: 赛季包购买状态 +- `deleted_at`: 删除时间 + +**业务逻辑**: +```sql +-- 角色付费状态 +case when purchase_season_package = '[1]' then 0 else 1 end as characer_pay_status + +-- 性别映射 +case when gender = 0 then 'girl' + when gender = 1 then 'boy' + else 'unknow' +end as gender + +-- 提取出生年份 +case when split_part(birthday,'-',1) = '' then '0000' + else split_part(birthday,'-',1) +end as birthday +``` + +--- + +## 课程播放记录表（分表） + +### 用户章节播放记录 +**表名**: `bi_user_chapter_play_record_0` ~ `bi_user_chapter_play_record_7` + +**说明**: 按分表存储，共8张表，需要使用 UNION ALL 合并 + +**关键字段**: +- `user_id`: 用户ID +- `chapter_id`: 章节ID +- `chapter_unique_id`: 完课唯一标识 +- `updated_at`: 更新时间 +- `play_status`: 播放状态（1表示完成） + +**常用筛选条件**: +```sql +where chapter_id in (55,56,57,58,59) -- 指定章节 + and play_status = 1 -- 播放完成 +``` + +--- + +### 用户组件播放记录 +**表名**: `bi_user_component_play_record_0` ~ `bi_user_component_play_record_7` + +**说明**: 按分表存储，共8张表，需要使用 UNION ALL 合并 + +**关键字段**: +- `chapter_unique_id`: 完课唯一标识 +- `interval_time`: 播放时长（毫秒） + +**业务逻辑**: +```sql +-- 计算完课耗时（mm:ss格式） +format('%s:%s', + floor(sum(interval_time)/1000/60), + mod((sum(interval_time)/1000),60) +) as finish_time +``` + +--- + +## 课程信息表 + +### 课程单元表 +**表名**: `bi_level_unit_lesson` + +**关键字段**: +- `id`: ID（关联 chapter_id） +- `course_level`: 课程级别 +- `course_season`: 课程赛季 +- `course_unit`: 课程单元 +- `course_lesson`: 课程课时 + +**业务逻辑**: +```sql +-- 生成课程ID +format('%s-%s-%s-%s', + course_level, + course_season, + course_unit, + course_lesson +) as course_id +``` + +--- + +## 其他表 + +### 账号登录表 +**表名**: `account_login` + +**关键字段**: +- `account_id`: 账号ID +- `login_date`: 登录日期 diff --git a/business_knowledge/feishu_format_rules.md b/business_knowledge/feishu_format_rules.md new file mode 100644 index 0000000..fb1a2b9 --- /dev/null +++ b/business_knowledge/feishu_format_rules.md @@ -0,0 +1,53 @@ +# 飞书文档排版规则 + +## 飞书文档块类型 + +根据观察，飞书文档的块类型： + +| block_type | 说明 | +|-----------|------| +| 1 | Page（页面）| +| 2 | Text（文本块）| +| 3 | Heading1（一级标题）| +| 4 | Heading2（二级标题）| +| 5 | Heading3（三级标题）| +| 6 | Bulleted List（无序列表）| +| 7 | Numbered List（有序列表）| +| 8 | To-do（待办事项）| +| 9 | Quote（引用）| +| 10 | Code（代码块）| +| 11 | Divider（分隔线）| +| 34 | Quote Container（引用容器）| + +## 排版最佳实践 + +### 1. 标题层级 +- 使用 Heading2/Heading3 来组织内容结构 +- 避免太多层级，保持清晰 + +### 2. 列表使用 +- 无序列表（type 6）用于列举项目 +- 有序列表（type 7）用于步骤说明 + +### 3. 分隔线 +- 使用 Divider（type 11）来分隔大的内容区块 + +### 4. 引用 +- 使用 Quote（type 9）或 Quote Container（type 34）来强调重要内容 + +### 5. 文本格式 +- 善用加粗、斜体等文本样式 +- 保持整体排版简洁美观 + +## 更新飞书文档的注意事项 + +⚠️ **重要：不要直接用 write 覆盖整个文档！** + +**推荐做法：** +1. 先用 list_blocks 查看当前文档结构 +2. 用 update_block 逐个更新需要修改的块 +3. 或者如果必须重写，要确保保持原来的块结构和格式 + +**避免：** +- ❌ 直接用 write 方法覆盖整个文档（会丢失所有格式） +- ❌ 把所有内容都放在一个 Text 块里 diff --git a/business_knowledge/fetch_wiki_docs.py b/business_knowledge/fetch_wiki_docs.py new file mode 100644 index 0000000..ea7f70f --- /dev/null +++ b/business_knowledge/fetch_wiki_docs.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +""" +批量读取飞书 Wiki 文档并保存到本地知识库 +""" + +import json +import os +from datetime import datetime + +# Wiki 子页面列表 +wiki_pages = [ + {"node_token": "O7QvwdY8piO8aUkhxYecA1qZnBe", "title": "全字段大表", "obj_token": "VVyWd5491o6tuqxceCVci6dVnFd"}, + {"node_token": "Y6Iywqf75iepbUkvJzLcfiUYnkg", "title": "平均通关时长", "obj_token": "EpP7d6h2SoaTyJx1lZRcXXdLnVe"}, + {"node_token": "KQihwMjO9i1zjFkqTgBcq67Snzc", "title": "新增注册用户数by渠道", "obj_token": "AzRPddp97o7To8x8VkxcFGr8nBh"}, + {"node_token": "Zt7RwfGLWiacslkO2glcheWjnwf", "title": "课程进入完成率", "obj_token": "PwIydfZcHo5eZgxi8XLcOtjOnSb"}, + {"node_token": "LTaiw3OmUi2pcckDWuNcyBIVnAd", "title": "账号角色年龄地址", "obj_token": "CUa2du2sSoNFSRxl3vFc8ucInEm"}, + {"node_token": "ZAPJwIODRiNYE5kTuNtcpSlvnIX", "title": "退费率", "obj_token": "DC1Qdhpitowt9lxxo1acEzOwnFc"}, + {"node_token": "Cb3KwPWLriG7GgkN73pcM0Idnch", "title": "销转学习进度", "obj_token": "G1p9dhK63oLWMzxyGQ8csZGMnDh"}, + {"node_token": "EBEiwQsw2iOtgekDldHcQxgwnOh", "title": "班主任关注数据", "obj_token": "NcVqdRKtrowglNxs9CocDekunje"}, + {"node_token": "BZPkwARxiixUZRk4BW9cij50nDe", "title": "端内GMV", "obj_token": "FkVCd1AruoD9xWxxVpzc16hinVh"}, + {"node_token": "AQpnwpsfOixYGtk4jf0c6t9XncG", "title": "端内用户课程进入完成率", "obj_token": "Ueu7dtgSHoNYfsxCDHmcY6E4nid"}, + {"node_token": "PyqEwXXqsiQybPkpGbscUjUFnOg", "title": "端内购课用户学习行为", "obj_token": "ZTxod4IUWo5yMexf8AHcBbpFnMg"}, + {"node_token": "OyXlwY2vyisvV1kc3HhcMyMVnTd", "title": "转化率", "obj_token": "ATJ0dfajQo5CSexQd8hc9i3pnWe"}, + {"node_token": "MWpZwV01fitaKjkCRSxckMUunRb", "title": "课程ID映射", "obj_token": "GenUdsXCloUdYhxMvxqcWBMdnhb"} +] + +def safe_filename(title): + """生成安全的文件名""" + return "".join(c for c in title if c.isalnum() or c in (' ', '-', '_')).rstrip().replace(' ', '_') + +def main(): + print("="*60) + print("飞书 Wiki 文档批量获取") + print("="*60) + + output_dir = "sql_queries" + os.makedirs(output_dir, exist_ok=True) + + print(f"\n共 {len(wiki_pages)} 个文档需要获取") + print(f"输出目录: {output_dir}") + + # 创建索引文件 + index_content = "# SQL 查询文档索引\n\n" + index_content += f"创建时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n" + index_content += "## 文档列表\n\n" + + for i, page in enumerate(wiki_pages, 1): + filename = safe_filename(page['title']) + ".md" + filepath = os.path.join(output_dir, filename) + + print(f"\n[{i}/{len(wiki_pages)}] 处理: {page['title']}") + print(f" 文件: {filepath}") + + # 创建占位文件 + with open(filepath, 'w', encoding='utf-8') as f: + f.write(f"# {page['title']}\n\n") + f.write(f"**获取时间:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n") + f.write(f"**飞书文档 Token:** {page['obj_token']}\n\n") + f.write(f"**注意:** 此文档需要通过 feishu_doc 工具读取完整内容\n\n") + f.write("---\n\n") + f.write("## 使用说明\n\n") + f.write("使用以下命令读取完整文档内容:\n\n") + f.write("```bash\n") + f.write(f"feishu_doc read {page['obj_token']}\n") + f.write("```\n") + + # 更新索引 + index_content += f"- [{page['title']}]({filename})\n" + + print(f" ✅ 已创建占位文件") + + # 写入索引文件 + with open(os.path.join(output_dir, "README.md"), 'w', encoding='utf-8') as f: + f.write(index_content) + + print("\n" + "="*60) + print("✅ 初始化完成") + print("="*60) + print("\n下一步: 使用 feishu_doc 工具逐个读取文档内容") + print("或者让我继续为你读取这些文档的完整内容") + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/CLAUDE.md b/business_knowledge/git_scripts/CLAUDE.md new file mode 100644 index 0000000..7fbbbf5 --- /dev/null +++ b/business_knowledge/git_scripts/CLAUDE.md @@ -0,0 +1,70 @@ +# 项目说明 + +## 项目概述 +用户数据提取和分析工具集，用于从各种数据源（ES、数据库等）导出和分析用户数据。 + +## 脚本列表 + +### export_realtime_asr.py +**功能**: 导出流式语音 ASR 数据 + +**版本**: v1.0 + +**数据源**: +- Elasticsearch 索引: `llm_realtime_asr_log` + +**配置说明**: +- 在脚本开头配置开始和结束日期（8位数字格式，如 20260101） +- ES 连接信息通过环境变量配置（需要创建 .env 文件） + +**依赖包**: +``` +elasticsearch +pandas +openpyxl +python-dotenv +``` + +**运行方式**: +```bash +python export_realtime_asr.py +``` + +**输出**: +- 输出目录: `output/` +- 文件命名: `realtime_asr_export_{开始日期}_{结束日期}.xlsx` +- Excel 列: voice_id, asr_prompt, result_str, timestamp, audio_url, source + +**数据处理逻辑**: +- 从 ES 使用 scroll API 分批读取数据（每批1000条） +- 按 voice_id 聚合，仅保留恰好有2条记录的 voice_id +- 取两条记录中最新的 timestamp +- 自动拼接 audio_url + +**特点**: +- 支持大数据量处理（几十万级别） +- 实时进度显示 +- 自动过滤异常数据（非2条记录的 voice_id） + +--- + +### 其他脚本 +- `export_user_id_data.py`: 用户ID数据导出 +- `batch_add_shengtong_result.py`: 批量添加声通评测结果 +- `shengtong_eval.py`: 声通评测 +- `calc_score_diff_stats.py`: 分数差异统计 +- `export_unit_summary.py`: 单元总结统计导出 + +## 环境配置 + +需要创建 `.env` 文件，包含以下配置: +``` +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx +``` + +## 最近更新 +- 2026-01-27: 新增 export_realtime_asr.py 脚本，支持流式语音 ASR 数据导出 diff --git a/business_knowledge/git_scripts/batch_add_shengtong_result.py b/business_knowledge/git_scripts/batch_add_shengtong_result.py new file mode 100644 index 0000000..8db5962 --- /dev/null +++ b/business_knowledge/git_scripts/batch_add_shengtong_result.py @@ -0,0 +1,853 @@ +""" +声通语音评测批量处理工具 + +功能说明: +- 读取 Excel 文件，其中包含音频链接（userAudio 字段）和参考文本（refText 字段） +- 调用声通 API 对音频进行评测，获取总分、明细和recordId +- 在原 Excel 中添加"测试总分"、"测试明细"和"测试recordId"三个字段 +- 输出文件命名为: {原文件名}_add_shengtong_result.xlsx +- 支持串行和并发两种处理模式 + +环境变量配置: +- ST_APP_KEY: 声通应用 Key +- ST_SECRET_KEY: 声通 Secret Key + +声通API文档: http://api.stkouyu.com +""" + +import pandas as pd +import os +import requests +import tempfile +from pathlib import Path +import json +import time +import hashlib +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading +from queue import Queue +import logging + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler('shengtong_batch_processing.log'), + logging.StreamHandler() + ] +) + +# 从 .env 文件加载环境变量 +from dotenv import load_dotenv +load_dotenv() + +# ==================== 全局配置 ==================== +# DEBUG 模式开关（控制详细日志输出） +DEBUG_MODE = False + + +def debug_print(message): + """ + DEBUG 信息输出函数 + + Args: + message (str): 要输出的调试信息 + """ + if DEBUG_MODE: + print(f"[DEBUG] {message}") + + +# ==================== 声通 API 相关代码 ==================== + +class ShengtongEvaluator: + """声通口语评测 API 封装类""" + + def __init__(self): + """从环境变量读取 API 配置""" + self.app_key = os.environ.get('ST_APP_KEY', '') + self.secret_key = os.environ.get('ST_SECRET_KEY', '') + self.api_url = "http://api.stkouyu.com:8080/sent.eval" + + # 检查环境变量是否配置 + if not all([self.app_key, self.secret_key]): + raise ValueError( + "请配置声通 API 环境变量: ST_APP_KEY, ST_SECRET_KEY" + ) + + def _generate_signature(self, data: str) -> str: + """生成SHA1签名""" + return hashlib.sha1(data.encode('utf-8')).hexdigest() + + def _build_request_params(self, ref_text: str, audio_ext: str) -> dict: + """构建请求参数""" + timestamp = str(int(time.time())) + user_id = str(uuid.uuid4()) + + # 生成签名 + connect_data = self.app_key + timestamp + self.secret_key + start_data = self.app_key + timestamp + user_id + self.secret_key + connect_sig = self._generate_signature(connect_data) + start_sig = self._generate_signature(start_data) + + # 构建请求参数 + params = { + "connect": { + "cmd": "connect", + "param": { + "sdk": { + "version": 16777472, + "source": 9, + "protocol": 2 + }, + "app": { + "applicationId": self.app_key, + "sig": connect_sig, + "timestamp": timestamp + } + } + }, + "start": { + "cmd": "start", + "param": { + "app": { + "applicationId": self.app_key, + "sig": start_sig, + "timestamp": timestamp, + "userId": user_id + }, + "audio": { + "audioType": audio_ext, + "channel": 1, + "sampleBytes": 2, + "sampleRate": 16000 + }, + "request": { + "coreType": "sent.eval", + "refText": ref_text, + "tokenId": "makee", + } + } + } + } + + return params + + def evaluate(self, audio_file_path: str, ref_text: str) -> dict: + """ + 调用声通API进行口语评测 + + Args: + audio_file_path (str): 音频文件路径 + ref_text (str): 参考文本 + + Returns: + dict: 评测结果 + """ + debug_print(f"开始评测音频文件: {audio_file_path}") + debug_print(f"评测文本: {ref_text}") + + # 检查音频文件是否存在 + if not os.path.exists(audio_file_path): + error_msg = f"音频文件不存在: {audio_file_path}" + logging.error(error_msg) + return {"error": error_msg} + + # 获取音频文件扩展名 + audio_ext = os.path.splitext(audio_file_path)[1][1:] # 去掉点号 + if not audio_ext: + audio_ext = "wav" # 默认为wav + + # 构建请求参数 + params = self._build_request_params(ref_text, audio_ext) + + # 读取音频文件 + try: + with open(audio_file_path, 'rb') as f: + audio_data = f.read() + + # 构建multipart/form-data请求 + files = { + 'text': (None, json.dumps(params)), + 'audio': (f"{int(time.time() * 1000000)}.{audio_ext}", audio_data) + } + + headers = { + 'Request-Index': '0' + } + + debug_print("开始发送请求到声通API...") + response = requests.post( + self.api_url, + files=files, + headers=headers, + timeout=30 + ) + + if response.status_code == 200: + result = response.json() + debug_print("声通API返回成功") + return result + else: + error_msg = f"请求失败，状态码: {response.status_code}" + logging.error(f"{error_msg}, 响应: {response.text}") + return { + "error": error_msg, + "response": response.text + } + + except requests.exceptions.RequestException as e: + error_msg = f"请求异常: {str(e)}" + logging.error(error_msg) + return {"error": error_msg} + except Exception as e: + error_msg = f"评测过程出错: {str(e)}" + logging.error(error_msg) + return {"error": error_msg} + + +def evaluate_audio_file(audio_file_path, text="nice to meet you."): + """ + 简化的音频评测函数 + + Args: + audio_file_path (str): 音频文件路径 + text (str): 评测文本内容 + + Returns: + dict: 评测结果JSON + """ + api = ShengtongEvaluator() + return api.evaluate(audio_file_path, text) + + +# ==================== 批量处理相关代码 ==================== + +def download_audio_file(audio_url, temp_dir, max_retries=3, timeout=30): + """ + 下载音频文件到临时目录（增强版本） + + Args: + audio_url (str): 音频文件URL + temp_dir (str): 临时目录路径 + max_retries (int): 最大重试次数 + timeout (int): 请求超时时间（秒） + + Returns: + str: 下载的音频文件路径，失败返回None + """ + if not audio_url or pd.isna(audio_url): + logging.warning("音频URL为空或无效") + return None + + # 从URL中提取文件名 + try: + file_name = os.path.basename(audio_url.split('?')[0]) # 去除URL参数 + if not file_name or '.' not in file_name: + file_name = f"audio_{hash(audio_url) % 100000}.wav" # 生成默认文件名 + + file_path = os.path.join(temp_dir, file_name) + + # 重试机制 + for attempt in range(max_retries): + try: + logging.info(f"正在下载音频文件 (尝试 {attempt + 1}/{max_retries}): {audio_url}") + + # 设置请求头，模拟浏览器 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + } + + response = requests.get(audio_url, timeout=timeout, headers=headers, stream=True) + response.raise_for_status() + + # 检查内容类型 + content_type = response.headers.get('content-type', '') + if not any(audio_type in content_type.lower() for audio_type in ['audio', 'wav', 'mp3', 'ogg', 'flac']): + logging.warning(f"可能不是音频文件，Content-Type: {content_type}") + + # 写入文件 + with open(file_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + # 验证文件大小 + file_size = os.path.getsize(file_path) + if file_size == 0: + raise ValueError("下载的文件为空") + + logging.info(f"音频文件下载成功: {file_path} (大小: {file_size} bytes)") + return file_path + + except requests.exceptions.Timeout: + logging.warning(f"下载超时 (尝试 {attempt + 1}/{max_retries}): {audio_url}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) # 指数退避 + continue + except requests.exceptions.RequestException as e: + logging.warning(f"下载请求异常 (尝试 {attempt + 1}/{max_retries}): {str(e)}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) + continue + except Exception as e: + logging.error(f"下载过程中发生未知错误 (尝试 {attempt + 1}/{max_retries}): {str(e)}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) + continue + + logging.error(f"音频文件下载失败，已达到最大重试次数: {audio_url}") + return None + + except Exception as e: + logging.error(f"下载音频文件时发生异常: {str(e)}") + return None + + +def format_shengtong_details(shengtong_result): + """ + 格式化声通评测结果为明细字符串 + + Args: + shengtong_result (dict): 声通API返回的结果 + + Returns: + str: 格式化的明细字符串 + """ + if not shengtong_result or 'error' in shengtong_result: + return "" + + try: + # 从result字段中获取words数组 + result = shengtong_result.get('result', {}) + words = result.get('words', []) + + if not words: + return "" + + details = [] + for word in words: + # 获取单词内容和得分 + word_text = word.get('word', '') + scores = word.get('scores', {}) + overall_score = scores.get('overall', 0) + + # 格式化为 "单词 分数" + details.append(f"{word_text} {int(overall_score)}") + + return "\n".join(details) + + except Exception as e: + logging.error(f"格式化声通明细失败: {str(e)}") + return "" + + +def get_shengtong_total_score(shengtong_result): + """ + 获取声通评测总分 + + Args: + shengtong_result (dict): 声通API返回的结果 + + Returns: + int: 总分，失败返回0 + """ + if not shengtong_result or 'error' in shengtong_result: + return 0 + + try: + result = shengtong_result.get('result', {}) + overall_score = result.get('overall', 0) + return int(overall_score) + except Exception as e: + logging.error(f"获取声通总分失败: {str(e)}") + return 0 + + +def get_shengtong_record_id(shengtong_result): + """ + 获取声通评测recordId + + Args: + shengtong_result (dict): 声通API返回的结果 + + Returns: + str: recordId，失败返回空字符串 + """ + if not shengtong_result or 'error' in shengtong_result: + return "" + + try: + record_id = shengtong_result.get('recordId', '') + return str(record_id) if record_id else "" + except Exception as e: + logging.error(f"获取声通recordId失败: {str(e)}") + return "" + + +def process_single_row(row_data, temp_dir, results_dict, lock, rate_limiter=None): + """ + 处理单行数据（并发版本，增强错误处理和时间分析） + + Args: + row_data (tuple): (index, row) 数据 + temp_dir (str): 临时目录路径 + results_dict (dict): 结果字典 + lock (threading.Lock): 线程锁 + rate_limiter (Queue): 速率限制器 + + Returns: + None + """ + index, row = row_data + start_time = time.time() + timing_info = {} + + try: + # 1. 速率限制等待时间 + rate_limit_start = time.time() + if rate_limiter: + rate_limiter.get() # 获取令牌 + timing_info['rate_limit_wait'] = time.time() - rate_limit_start + + logging.info(f"开始处理第 {index + 1} 行数据") + + # 2. 数据预处理时间 + preprocess_start = time.time() + ref_text = str(row['refText']) if pd.notna(row['refText']) else "" + audio_url = str(row['userAudio']) if pd.notna(row['userAudio']) else "" + + # 数据验证 + if not ref_text: + raise ValueError("refText 为空或无效") + + if not audio_url: + raise ValueError("userAudio 为空或无效") + timing_info['preprocess'] = time.time() - preprocess_start + + # 3. 音频下载时间 + download_start = time.time() + audio_file_path = download_audio_file(audio_url, temp_dir) + timing_info['audio_download'] = time.time() - download_start + + if not audio_file_path: + raise ValueError("音频文件下载失败") + + try: + # 4. 声通API调用时间 + api_start = time.time() + logging.info(f"正在调用声通API评测: {ref_text}") + shengtong_result = evaluate_audio_file(audio_file_path, ref_text) + timing_info['api_call'] = time.time() - api_start + + if not shengtong_result: + raise ValueError("声通API返回空结果") + + # 5. 结果处理时间 + result_process_start = time.time() + shengtong_details = format_shengtong_details(shengtong_result) + shengtong_total_score = get_shengtong_total_score(shengtong_result) + shengtong_record_id = get_shengtong_record_id(shengtong_result) + timing_info['result_process'] = time.time() - result_process_start + + # 6. 数据更新时间 + update_start = time.time() + with lock: + results_dict[index] = { + '测试总分': shengtong_total_score, + '测试明细': shengtong_details, + '测试recordId': shengtong_record_id + } + timing_info['data_update'] = time.time() - update_start + + # 计算总耗时 + total_time = time.time() - start_time + timing_info['total'] = total_time + + # 详细的时间分析日志 + logging.info(f"第 {index + 1} 行处理成功 - 总分: {shengtong_total_score} | " + f"总耗时: {total_time:.2f}s | " + f"速率等待: {timing_info['rate_limit_wait']:.2f}s | " + f"预处理: {timing_info['preprocess']:.3f}s | " + f"音频下载: {timing_info['audio_download']:.2f}s | " + f"API调用: {timing_info['api_call']:.2f}s | " + f"结果处理: {timing_info['result_process']:.3f}s | " + f"数据更新: {timing_info['data_update']:.3f}s") + + except Exception as api_error: + total_time = time.time() - start_time + logging.error(f"第 {index + 1} 行声通API调用失败: {str(api_error)} | " + f"总耗时: {total_time:.2f}s | " + f"音频下载: {timing_info.get('audio_download', 0):.2f}s | " + f"API调用: {timing_info.get('api_call', 0):.2f}s") + with lock: + results_dict[index] = { + '测试总分': 0, + '测试明细': "", + '测试recordId': "", + 'error': f'API调用失败: {str(api_error)}' + } + + finally: + # 7. 清理时间 + cleanup_start = time.time() + try: + if audio_file_path and os.path.exists(audio_file_path): + os.remove(audio_file_path) + logging.debug(f"已删除临时文件: {audio_file_path}") + except Exception as cleanup_error: + logging.warning(f"清理临时文件失败: {str(cleanup_error)}") + timing_info['cleanup'] = time.time() - cleanup_start + + # 释放速率限制令牌 + if rate_limiter: + try: + rate_limiter.put(None, timeout=1) # 归还令牌 + except: + pass # 队列可能已满，忽略 + + except Exception as e: + total_time = time.time() - start_time + logging.error(f"第 {index + 1} 行处理异常: {str(e)} | 总耗时: {total_time:.2f}s") + with lock: + results_dict[index] = { + '测试总分': 0, + '测试明细': "", + '测试recordId': "", + 'error': f'处理异常: {str(e)}' + } + + # 释放速率限制令牌 + if rate_limiter: + try: + rate_limiter.put(None, timeout=1) + except: + pass + + +def process_excel_with_shengtong_concurrent(input_file_path, output_dir="output/audio", max_workers=3, rate_limit_per_second=3): + """ + 处理Excel文件，添加声通评测结果（并发版本，增强控制） + + Args: + input_file_path (str): 输入Excel文件路径 + output_dir (str): 输出目录路径，默认为 output/audio + max_workers (int): 最大并发线程数，默认3 + rate_limit_per_second (int): 每秒最大请求数，默认3 + + Returns: + bool: 处理是否成功 + """ + start_time = time.time() + + try: + # 读取Excel文件 + logging.info(f"正在读取Excel文件: {input_file_path}") + df = pd.read_excel(input_file_path) + + # 检查必要的列是否存在 + required_columns = ['refText', 'userAudio'] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + logging.error(f"Excel文件缺少必要的列: {missing_columns}") + return False + + # 数据预处理和验证 + total_rows = len(df) + valid_rows = 0 + for index, row in df.iterrows(): + if pd.notna(row.get('refText')) and pd.notna(row.get('userAudio')): + valid_rows += 1 + + logging.info(f"总行数: {total_rows}, 有效行数: {valid_rows}") + + if valid_rows == 0: + logging.warning("没有找到有效的数据行") + return False + + # 添加新列 + df['测试总分'] = 0 + df['测试明细'] = "" + df['测试recordId'] = "" + + # 创建优化的速率限制器 + effective_rate_limit = max(rate_limit_per_second, max_workers) + rate_limiter = Queue(maxsize=effective_rate_limit * 2) + + # 预填充令牌 + for _ in range(effective_rate_limit): + rate_limiter.put(None) + + # 启动优化的速率限制器补充线程 + def rate_limiter_refill(): + interval = 1.0 / effective_rate_limit + while True: + time.sleep(interval) + try: + rate_limiter.put(None, block=False) + except: + pass + + rate_thread = threading.Thread(target=rate_limiter_refill, daemon=True) + rate_thread.start() + + logging.info(f"速率限制设置: {effective_rate_limit} req/s (原始: {rate_limit_per_second}, 队列大小: {effective_rate_limit * 2})") + + # 创建临时目录用于下载音频文件 + with tempfile.TemporaryDirectory() as temp_dir: + logging.info(f"创建临时目录: {temp_dir}") + logging.info(f"开始并发处理，最大并发数: {max_workers}, 有效速率限制: {effective_rate_limit} req/s") + + # 准备数据 + row_data_list = [(index, row) for index, row in df.iterrows()] + + # 创建结果字典和线程锁 + results_dict = {} + lock = threading.Lock() + + # 使用线程池进行并发处理 + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # 提交所有任务 + future_to_index = { + executor.submit(process_single_row, row_data, temp_dir, results_dict, lock, rate_limiter): row_data[0] + for row_data in row_data_list + } + + # 等待任务完成并显示进度 + completed_count = 0 + success_count = 0 + error_count = 0 + + for future in as_completed(future_to_index): + completed_count += 1 + index = future_to_index[future] + + try: + future.result() # 获取结果，如果有异常会抛出 + + # 检查处理结果 + with lock: + result = results_dict.get(index, {}) + if result.get('error') is None: + success_count += 1 + else: + error_count += 1 + + # 显示进度 + if completed_count % 10 == 0 or completed_count == total_rows: + elapsed_time = time.time() - start_time + avg_time_per_item = elapsed_time / completed_count + remaining_time = avg_time_per_item * (total_rows - completed_count) + + logging.info(f"进度: {completed_count}/{total_rows} ({completed_count/total_rows*100:.1f}%) " + f"成功: {success_count}, 失败: {error_count}, " + f"预计剩余时间: {remaining_time:.1f}秒") + + except Exception as e: + error_count += 1 + logging.error(f"任务 {index + 1} 执行异常: {str(e)}") + with lock: + if index not in results_dict: + results_dict[index] = { + '测试总分': 0, + '测试明细': "", + '测试recordId': "", + 'error': f'任务执行异常: {str(e)}' + } + + # 将结果更新到DataFrame + logging.info("正在更新结果到DataFrame...") + for index in results_dict: + result = results_dict[index] + df.at[index, '测试总分'] = result.get('测试总分', 0) + df.at[index, '测试明细'] = result.get('测试明细', "") + df.at[index, '测试recordId'] = result.get('测试recordId', "") + + # 如果有错误，可以选择记录到备注列（如果存在） + if result.get('error') and '备注' in df.columns: + existing_note = str(df.at[index, '备注']) if pd.notna(df.at[index, '备注']) else "" + error_note = f"声通API错误: {result['error']}" + df.at[index, '备注'] = f"{existing_note}\n{error_note}".strip() + + # 创建输出目录 + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # 生成输出文件路径 + input_path = Path(input_file_path) + output_file_path = output_path / f"{input_path.stem}_add_shengtong_result.xlsx" + + # 保存结果 + logging.info(f"正在保存结果到: {output_file_path}") + df.to_excel(output_file_path, index=False) + + # 计算总耗时 + total_time = time.time() - start_time + + # 统计处理结果 + final_success_count = sum(1 for result in results_dict.values() if result.get('error') is None) + final_error_count = len(results_dict) - final_success_count + + logging.info("=" * 50) + logging.info("并发处理完成！") + logging.info(f"处理统计: 成功 {final_success_count} 条，失败 {final_error_count} 条，总计 {len(results_dict)} 条") + logging.info(f"总耗时: {total_time:.2f} 秒") + logging.info(f"平均处理时间: {total_time/len(results_dict):.2f} 秒/条") + logging.info(f"输出文件: {output_file_path}") + logging.info("=" * 50) + + return True + + except Exception as e: + logging.error(f"处理Excel文件时出错: {str(e)}") + return False + + +def process_excel_with_shengtong(input_file_path, output_dir="output/audio"): + """ + 处理Excel文件，添加声通评测结果（串行版本） + + Args: + input_file_path (str): 输入Excel文件路径 + output_dir (str): 输出目录路径，默认为 output/audio + + Returns: + bool: 处理是否成功 + """ + try: + # 读取Excel文件 + print(f"正在读取Excel文件: {input_file_path}") + df = pd.read_excel(input_file_path) + + # 检查必要的列是否存在 + required_columns = ['refText', 'userAudio'] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + print(f"错误: Excel文件缺少必要的列: {missing_columns}") + return False + + # 添加新列 + df['测试总分'] = 0 + df['测试明细'] = "" + df['测试recordId'] = "" + + # 创建临时目录用于下载音频文件 + with tempfile.TemporaryDirectory() as temp_dir: + print(f"创建临时目录: {temp_dir}") + + # 处理每一行数据 + total_rows = len(df) + for index, row in df.iterrows(): + print(f"\n处理进度: {index + 1}/{total_rows}") + + ref_text = str(row['refText']) if pd.notna(row['refText']) else "" + audio_url = str(row['userAudio']) if pd.notna(row['userAudio']) else "" + + if not ref_text or not audio_url: + print(f"第 {index + 1} 行数据不完整，跳过") + continue + + print(f"参考文本: {ref_text}") + print(f"音频URL: {audio_url}") + + # 下载音频文件 + audio_file_path = download_audio_file(audio_url, temp_dir) + if not audio_file_path: + print(f"第 {index + 1} 行音频下载失败，跳过") + continue + + # 调用声通API进行评测 + print("正在调用声通API进行评测...") + try: + shengtong_result = evaluate_audio_file(audio_file_path, ref_text) + print(f"声通API返回结果: {json.dumps(shengtong_result, indent=2, ensure_ascii=False)}") + + # 提取总分、明细和recordId + total_score = get_shengtong_total_score(shengtong_result) + details = format_shengtong_details(shengtong_result) + record_id = get_shengtong_record_id(shengtong_result) + + # 更新DataFrame + df.at[index, '测试总分'] = total_score + df.at[index, '测试明细'] = details + df.at[index, '测试recordId'] = record_id + + print(f"测试总分: {total_score}") + print(f"测试明细: {details}") + print(f"测试recordId: {record_id}") + + except Exception as e: + print(f"第 {index + 1} 行声通API调用失败: {str(e)}") + continue + + # 删除临时音频文件 + try: + os.remove(audio_file_path) + except: + pass + + # 添加延时避免API调用过于频繁 + time.sleep(1) + + # 创建输出目录 + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # 生成输出文件路径 + input_path = Path(input_file_path) + output_file_path = output_path / f"{input_path.stem}_add_shengtong_result.xlsx" + + # 保存结果 + print(f"\n正在保存结果到: {output_file_path}") + df.to_excel(output_file_path, index=False) + print("处理完成！") + + return True + + except Exception as e: + print(f"处理Excel文件时出错: {str(e)}") + return False + + +if __name__ == "__main__": + # ==================== 配置参数 ==================== + input_file = "人工筛选测试集v2_denoise.xlsx" + output_directory = "output/audio" # 输出目录，可以修改 + use_concurrent = True # True: 使用并发版本，False: 使用串行版本 + + # DEBUG 模式开关（True: 显示详细调试信息，False: 仅显示关键信息） + enable_debug = False # 可以设置为 True 来查看详细的 DEBUG 日志 + + # 设置全局 DEBUG_MODE + globals()['DEBUG_MODE'] = enable_debug + + # 检查环境变量 + required_env_vars = ['ST_APP_KEY', 'ST_SECRET_KEY'] + missing_vars = [var for var in required_env_vars if not os.environ.get(var)] + + if missing_vars: + print(f"错误: 缺少必要的环境变量: {missing_vars}") + print("请在 .env 文件或系统环境变量中配置:") + print(" ST_APP_KEY=你的应用Key") + print(" ST_SECRET_KEY=你的Secret Key") + elif not os.path.exists(input_file): + print(f"文件不存在: {input_file}") + print("请确保Excel文件存在并包含 'refText' 和 'userAudio' 列") + else: + if use_concurrent: + print("使用并发版本处理（3路并发，3 req/s）...") + success = process_excel_with_shengtong_concurrent( + input_file, + output_dir=output_directory, + max_workers=3, + rate_limit_per_second=3 + ) + else: + print("使用串行版本处理...") + success = process_excel_with_shengtong(input_file, output_dir=output_directory) + + if success: + print("处理成功！") + else: + print("处理失败！") diff --git a/business_knowledge/git_scripts/batch_add_xunfei_result.py b/business_knowledge/git_scripts/batch_add_xunfei_result.py new file mode 100644 index 0000000..3e07493 --- /dev/null +++ b/business_knowledge/git_scripts/batch_add_xunfei_result.py @@ -0,0 +1,1090 @@ +""" +讯飞语音评测批量处理工具 + +功能说明: +- 读取 Excel 文件，其中包含音频链接（userAudio 字段）和参考文本（refText 字段） +- 调用讯飞 API 对音频进行评测，获取总分和明细 +- 在原 Excel 中添加"讯飞总分"和"讯飞明细"两个字段 +- 输出文件命名为: {原文件名}_add_xunfei_result.xlsx +- 支持串行和并发两种处理模式 + +环境变量配置: +- XUNFEI_APPID: 讯飞应用 ID +- XUNFEI_API_SECRET: 讯飞 API 密钥 +- XUNFEI_API_KEY: 讯飞 API Key + +讯飞技术文档: https://www.xfyun.cn/doc/Ise/IseAPI.html +""" + +import pandas as pd +import os +import requests +import tempfile +from pathlib import Path +import json +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading +from queue import Queue +import logging +import websocket +import datetime +import hashlib +import base64 +import hmac +from urllib.parse import urlencode +import ssl +from wsgiref.handlers import format_date_time +from datetime import datetime +from time import mktime +import xml.etree.ElementTree as ET + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler('xunfei_batch_processing.log'), + logging.StreamHandler() + ] +) + +# 从 .env 文件加载环境变量 +from dotenv import load_dotenv +load_dotenv() + +# ==================== 全局配置 ==================== +# DEBUG 模式开关（控制详细日志输出） +DEBUG_MODE = False + + +def debug_print(message): + """ + DEBUG 信息输出函数 + + Args: + message (str): 要输出的调试信息 + """ + if DEBUG_MODE: + print(f"[DEBUG] {message}") + + +# ==================== 讯飞 API 相关代码 ==================== + +class XunfeiISEAPI: + """讯飞语音评测 API 封装类""" + + def __init__(self): + """从环境变量读取 API 配置""" + self.host_url = "ws://ise-api.xfyun.cn/v2/open-ise" + self.appid = os.environ.get('XUNFEI_APPID', '') + self.api_secret = os.environ.get('XUNFEI_API_SECRET', '') + self.api_key = os.environ.get('XUNFEI_API_KEY', '') + + # 检查环境变量是否配置 + if not all([self.appid, self.api_secret, self.api_key]): + raise ValueError( + "请配置讯飞 API 环境变量: XUNFEI_APPID, XUNFEI_API_SECRET, XUNFEI_API_KEY" + ) + + self.result = None + self.error = None + + def _detect_audio_format(self, audio_file_path): + """检测音频文件格式""" + try: + # 通过文件扩展名检测 + file_ext = os.path.splitext(audio_file_path)[1].lower() + if file_ext == '.wav': + return 'wav' + elif file_ext == '.mp3': + return 'mp3' + + # 通过文件头检测 + with open(audio_file_path, 'rb') as f: + header = f.read(12) + if len(header) >= 12: + # WAV文件头: RIFF....WAVE + if header[:4] == b'RIFF' and header[8:12] == b'WAVE': + return 'wav' + # MP3文件头: ID3 或 0xFF 0xFB/0xFA + elif header[:3] == b'ID3' or (header[0] == 0xFF and (header[1] & 0xE0) == 0xE0): + return 'mp3' + + # 默认返回wav + return 'wav' + except Exception as e: + print(f"[WARNING] 音频格式检测失败: {str(e)}, 默认使用WAV格式") + return 'wav' + + def _remove_wav_header(self, audio_file_path): + """去除WAV文件头部，返回纯音频数据""" + try: + with open(audio_file_path, 'rb') as f: + # 读取WAV文件头 + riff_header = f.read(12) # RIFF header (12 bytes) + if len(riff_header) < 12 or riff_header[:4] != b'RIFF' or riff_header[8:12] != b'WAVE': + print(f"[WARNING] 不是有效的WAV文件，直接返回原始数据") + f.seek(0) + return f.read() + + # 跳过format chunk + while True: + chunk_header = f.read(8) + if len(chunk_header) < 8: + break + + chunk_id = chunk_header[:4] + chunk_size = int.from_bytes(chunk_header[4:8], byteorder='little') + + if chunk_id == b'data': + # 找到data chunk，返回音频数据 + audio_data = f.read(chunk_size) + debug_print(f"WAV头部已去除，音频数据大小: {len(audio_data)} bytes") + return audio_data + else: + # 跳过其他chunk + f.seek(chunk_size, 1) + if chunk_size % 2: # 如果chunk大小是奇数，需要跳过一个字节对齐 + f.seek(1, 1) + + # 如果没找到data chunk，返回从当前位置开始的所有数据 + print(f"[WARNING] 未找到data chunk，返回剩余数据") + return f.read() + + except Exception as e: + print(f"[ERROR] WAV头部处理失败: {str(e)}, 返回原始文件数据") + with open(audio_file_path, 'rb') as f: + return f.read() + + def _generate_url(self): + """生成WebSocket连接URL""" + now_time = datetime.now() + now_date = format_date_time(mktime(now_time.timetuple())) + + # 拼接鉴权原始字符串 + origin_base = "host: " + "ise-api.xfyun.cn" + "\n" + origin_base += "date: " + now_date + "\n" + origin_base += "GET " + "/v2/open-ise " + "HTTP/1.1" + + # sha256加密 + signature_sha = hmac.new(self.api_secret.encode('utf-8'), origin_base.encode('utf-8'), + digestmod=hashlib.sha256).digest() + signature_sha = base64.b64encode(signature_sha).decode(encoding='utf-8') + + authorization_origin = "api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"" % ( + self.api_key, "hmac-sha256", "host date request-line", signature_sha) + authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8') + + # 将请求的鉴权参数组合为字典 + dict_data = { + "authorization": authorization, + "date": now_date, + "host": "ise-api.xfyun.cn" + } + ws_url = self.host_url + '?' + urlencode(dict_data) + return ws_url + + def _on_message(self, ws, message): + """处理WebSocket消息""" + try: + debug_print(f"收到消息: {message}") + response = json.loads(message) + debug_print(f"解析后的响应: {json.dumps(response, indent=2, ensure_ascii=False)}") + + # 检查响应结构 + if "data" not in response: + print(f"[ERROR] 响应中缺少 'data' 字段") + self.error = f"响应格式错误: 缺少 'data' 字段" + ws.close() + return + + data = response["data"] + if "status" not in data: + print(f"[ERROR] data 中缺少 'status' 字段") + self.error = f"响应格式错误: 缺少 'status' 字段" + ws.close() + return + + status = data["status"] + debug_print(f"状态码: {status}") + + if status == 2: # 评测完成 + if "data" not in data: + print(f"[ERROR] data 中缺少评测结果数据") + self.error = f"响应格式错误: 缺少评测结果数据" + ws.close() + return + + xml_data = base64.b64decode(data["data"]) + xml_string = xml_data.decode("utf-8") + debug_print(f"解码后的XML: {xml_string}") + self.result = self._parse_xml_result(xml_string) + debug_print(f"解析后的结果: {json.dumps(self.result, indent=2, ensure_ascii=False)}") + ws.close() + except json.JSONDecodeError as e: + print(f"[ERROR] JSON解析失败: {str(e)}") + print(f"[ERROR] 原始消息: {message}") + self.error = f"JSON解析错误: {str(e)}" + ws.close() + except Exception as e: + print(f"[ERROR] 消息处理异常: {str(e)}") + print(f"[ERROR] 异常类型: {type(e).__name__}") + print(f"[ERROR] 原始消息: {message}") + self.error = f"消息处理错误: {str(e)}" + ws.close() + + def _on_error(self, ws, error): + """处理WebSocket错误""" + print(f"[ERROR] WebSocket错误: {str(error)}") + print(f"[ERROR] 错误类型: {type(error).__name__}") + self.error = f"WebSocket错误: {str(error)}" + + def _on_close(self, ws, reason, res): + """WebSocket连接关闭""" + debug_print(f"WebSocket连接关闭 - 原因: {reason}, 响应: {res}") + pass + + def _on_open(self, ws, audio_file, text="nice to meet you."): + """WebSocket连接打开，发送音频数据""" + try: + debug_print("WebSocket连接已打开") + debug_print(f"音频文件: {audio_file}") + debug_print(f"评测文本: {text}") + + # 检测音频格式 + audio_format = self._detect_audio_format(audio_file) + debug_print(f"检测到音频格式: {audio_format}") + + # 根据音频格式设置aue参数 + if audio_format == 'wav': + aue_param = "raw" # WAV文件使用raw + else: # mp3 + aue_param = "lame" # MP3文件使用lame + + debug_print(f"使用aue参数: {aue_param}") + + # 发送初始配置 + send_dict = { + "common": { + "app_id": self.appid + }, + "business": { + "category": "read_sentence", + "rstcd": "utf8", + "sub": "ise", + "group": "pupil", + "ent": "en_vip", + "tte": "utf-8", + "cmd": "ssb", + "auf": "audio/L16;rate=16000", + "aue": aue_param, + "text": '\uFEFF' + f"[content]\n{text}", + "ise_unite": "1", + "extra_ability": "pitch" + }, + "data": { + "status": 0, + "data": "" + } + } + debug_print(f"发送初始配置: {json.dumps(send_dict, indent=2, ensure_ascii=False)}") + ws.send(json.dumps(send_dict)) + + # 根据音频格式处理音频数据 + if audio_format == 'wav': + # WAV文件需要去除头部 + audio_data = self._remove_wav_header(audio_file) + debug_print(f"WAV文件头部已去除，音频数据大小: {len(audio_data)} bytes") + else: + # MP3文件直接读取 + with open(audio_file, "rb") as f: + audio_data = f.read() + debug_print(f"MP3文件直接读取，音频数据大小: {len(audio_data)} bytes") + + # 优化音频发送逻辑 + frame_count = 0 + data_size = len(audio_data) + + # 根据数据大小动态调整缓冲区大小和延迟 + if data_size > 50000: # 大于50KB的数据使用更大的缓冲区 + buffer_size = 12800 # 20倍缓冲区 + sleep_time = 0.02 # 减少延迟到20ms + else: + buffer_size = 1280 # 原始缓冲区 + sleep_time = 0.01 # 小文件使用更小延迟 + + debug_print(f"使用缓冲区大小: {buffer_size}, 延迟: {sleep_time}s") + + # 发送音频数据 + offset = 0 + while offset < data_size: + # 读取缓冲区大小的数据 + buffer = audio_data[offset:offset + buffer_size] + offset += len(buffer) + + if offset >= data_size: + # 发送最后一帧 + my_dict = { + "business": {"cmd": "auw", "aus": 4, "aue": aue_param}, + "data": {"status": 2, "data": str(base64.b64encode(buffer).decode())} + } + debug_print("发送最后一帧") + ws.send(json.dumps(my_dict)) + break + + # 发送中间帧 + send_dict = { + "business": { + "cmd": "auw", + "aus": 1, + "aue": aue_param + }, + "data": { + "status": 1, + "data": str(base64.b64encode(buffer).decode()), + "data_type": 1, + "encoding": "raw" + } + } + frame_count += 1 + if frame_count % 20 == 0: # 减少日志频率 + debug_print(f"已发送 {frame_count} 帧音频数据") + ws.send(json.dumps(send_dict)) + time.sleep(sleep_time) # 使用动态延迟 + + debug_print(f"音频发送完成，总共发送 {frame_count} 帧") + + except Exception as e: + print(f"[ERROR] 音频发送异常: {str(e)}") + print(f"[ERROR] 异常类型: {type(e).__name__}") + self.error = f"音频发送错误: {str(e)}" + ws.close() + + def _parse_xml_result(self, xml_string): + """解析XML评测结果""" + try: + root = ET.fromstring(xml_string) + + result = { + "total_score": 0, + "words": [], + "sentences": [] + } + + # 解析句子级别评分 + for sentence in root.findall('.//sentence'): + sentence_info = { + "content": sentence.get('content', ''), + "total_score": float(sentence.get('total_score', 0)), + "fluency_score": float(sentence.get('fluency_score', 0)), + "integrity_score": float(sentence.get('integrity_score', 0)), + "phone_score": float(sentence.get('phone_score', 0)) + } + result["sentences"].append(sentence_info) + result["total_score"] = sentence_info["total_score"] + + # 解析单词级别评分 + for word in root.findall('.//word'): + word_info = { + "content": word.get('content', ''), + "total_score": float(word.get('total_score', 0)), + "dp_message": int(word.get('dp_message', 0)), + "time_len": int(word.get('time_len', 0)), + "syllables": [] + } + + # 解析音节评分 + for syllable in word.findall('.//syllable'): + syllable_info = { + "content": syllable.get('content', ''), + "total_score": float(syllable.get('total_score', 0)), + "phones": [] + } + + # 解析音素评分 + for phone in syllable.findall('.//phone'): + phone_info = { + "content": phone.get('content', ''), + "total_score": float(phone.get('total_score', 0)), + "dp_message": int(phone.get('dp_message', 0)) + } + syllable_info["phones"].append(phone_info) + + word_info["syllables"].append(syllable_info) + + result["words"].append(word_info) + + return result + + except Exception as e: + return {"error": f"XML解析错误: {str(e)}"} + + def evaluate_audio(self, audio_file_path, text="nice to meet you.", timeout=30): + """ + 评测音频文件 + + Args: + audio_file_path (str): 音频文件路径 + text (str): 评测文本内容 + timeout (int): 超时时间（秒） + + Returns: + dict: 评测结果JSON + """ + debug_print(f"开始评测音频文件: {audio_file_path}") + debug_print(f"评测文本: {text}") + + # 检查音频文件是否存在 + if not os.path.exists(audio_file_path): + error_msg = f"音频文件不存在: {audio_file_path}" + print(f"[ERROR] {error_msg}") + return {"error": error_msg} + + # 重置结果 + self.result = None + self.error = None + + try: + # 生成WebSocket URL + ws_url = self._generate_url() + debug_print(f"WebSocket URL: {ws_url}") + + # 创建WebSocket连接 + websocket.enableTrace(False) + ws = websocket.WebSocketApp( + ws_url, + on_message=self._on_message, + on_error=self._on_error, + on_close=self._on_close, + on_open=lambda ws: self._on_open(ws, audio_file_path, text) + ) + + debug_print("开始WebSocket连接...") + # 运行WebSocket连接 + ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE}) + + debug_print("WebSocket连接结束") + # 返回结果 + if self.error: + print(f"[ERROR] 评测失败: {self.error}") + return {"error": self.error} + elif self.result: + debug_print("评测成功") + return self.result + else: + error_msg = "未收到评测结果" + print(f"[ERROR] {error_msg}") + return {"error": error_msg} + + except Exception as e: + error_msg = f"评测过程出错: {str(e)}" + print(f"[ERROR] {error_msg}") + print(f"[ERROR] 异常类型: {type(e).__name__}") + return {"error": error_msg} + + +def evaluate_audio_file(audio_file_path, text="nice to meet you."): + """ + 简化的音频评测函数 + + Args: + audio_file_path (str): 音频文件路径 + text (str): 评测文本内容 + + Returns: + dict: 评测结果JSON + """ + api = XunfeiISEAPI() + return api.evaluate_audio(audio_file_path, text) + + +# ==================== 批量处理相关代码 ==================== + +def download_audio_file(audio_url, temp_dir, max_retries=3, timeout=30): + """ + 下载音频文件到临时目录（增强版本） + + Args: + audio_url (str): 音频文件URL + temp_dir (str): 临时目录路径 + max_retries (int): 最大重试次数 + timeout (int): 请求超时时间（秒） + + Returns: + str: 下载的音频文件路径，失败返回None + """ + if not audio_url or pd.isna(audio_url): + logging.warning("音频URL为空或无效") + return None + + # 从URL中提取文件名 + try: + file_name = os.path.basename(audio_url.split('?')[0]) # 去除URL参数 + if not file_name or '.' not in file_name: + file_name = f"audio_{hash(audio_url) % 100000}.wav" # 生成默认文件名 + + file_path = os.path.join(temp_dir, file_name) + + # 重试机制 + for attempt in range(max_retries): + try: + logging.info(f"正在下载音频文件 (尝试 {attempt + 1}/{max_retries}): {audio_url}") + + # 设置请求头，模拟浏览器 + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' + } + + response = requests.get(audio_url, timeout=timeout, headers=headers, stream=True) + response.raise_for_status() + + # 检查内容类型 + content_type = response.headers.get('content-type', '') + if not any(audio_type in content_type.lower() for audio_type in ['audio', 'wav', 'mp3', 'ogg', 'flac']): + logging.warning(f"可能不是音频文件，Content-Type: {content_type}") + + # 写入文件 + with open(file_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) + + # 验证文件大小 + file_size = os.path.getsize(file_path) + if file_size == 0: + raise ValueError("下载的文件为空") + + logging.info(f"音频文件下载成功: {file_path} (大小: {file_size} bytes)") + return file_path + + except requests.exceptions.Timeout: + logging.warning(f"下载超时 (尝试 {attempt + 1}/{max_retries}): {audio_url}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) # 指数退避 + continue + except requests.exceptions.RequestException as e: + logging.warning(f"下载请求异常 (尝试 {attempt + 1}/{max_retries}): {str(e)}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) + continue + except Exception as e: + logging.error(f"下载过程中发生未知错误 (尝试 {attempt + 1}/{max_retries}): {str(e)}") + if attempt < max_retries - 1: + time.sleep(2 ** attempt) + continue + + logging.error(f"音频文件下载失败，已达到最大重试次数: {audio_url}") + return None + + except Exception as e: + logging.error(f"下载音频文件时发生异常: {str(e)}") + return None + + +def format_xunfei_details(xunfei_result): + """ + 格式化讯飞评测结果为明细字符串 + + Args: + xunfei_result (dict): 讯飞API返回的结果 + + Returns: + str: 格式化的明细字符串 + """ + if not xunfei_result or 'error' in xunfei_result: + return "" + + try: + words = xunfei_result.get('words', []) + if not words: + return "" + + details = [] + for word in words: + content = word.get('content', '') + total_score = word.get('total_score', 0) + details.append(f"{content} {int(total_score)}") + + return "\n".join(details) + + except Exception as e: + print(f"格式化讯飞明细失败: {str(e)}") + return "" + + +def get_xunfei_total_score(xunfei_result): + """ + 获取讯飞评测总分 + + Args: + xunfei_result (dict): 讯飞API返回的结果 + + Returns: + int: 总分，失败返回0 + """ + if not xunfei_result or 'error' in xunfei_result: + return 0 + + try: + return int(xunfei_result.get('total_score', 0)) + except Exception as e: + print(f"获取讯飞总分失败: {str(e)}") + return 0 + + +def process_single_row(row_data, temp_dir, results_dict, lock, rate_limiter=None): + """ + 处理单行数据（并发版本，增强错误处理和时间分析） + + Args: + row_data (tuple): (index, row) 数据 + temp_dir (str): 临时目录路径 + results_dict (dict): 结果字典 + lock (threading.Lock): 线程锁 + rate_limiter (Queue): 速率限制器 + + Returns: + None + """ + index, row = row_data + start_time = time.time() + timing_info = {} + + try: + # 1. 速率限制等待时间 + rate_limit_start = time.time() + if rate_limiter: + rate_limiter.get() # 获取令牌 + timing_info['rate_limit_wait'] = time.time() - rate_limit_start + + logging.info(f"开始处理第 {index + 1} 行数据") + + # 2. 数据预处理时间 + preprocess_start = time.time() + ref_text = str(row['refText']) if pd.notna(row['refText']) else "" + audio_url = str(row['userAudio']) if pd.notna(row['userAudio']) else "" + + # 数据验证 + if not ref_text: + raise ValueError("refText 为空或无效") + + if not audio_url: + raise ValueError("userAudio 为空或无效") + timing_info['preprocess'] = time.time() - preprocess_start + + # 3. 音频下载时间 + download_start = time.time() + audio_file_path = download_audio_file(audio_url, temp_dir) + timing_info['audio_download'] = time.time() - download_start + + if not audio_file_path: + raise ValueError("音频文件下载失败") + + try: + # 4. 讯飞API调用时间 + api_start = time.time() + logging.info(f"正在调用讯飞API评测: {ref_text}") + xunfei_result = evaluate_audio_file(audio_file_path, ref_text) + timing_info['api_call'] = time.time() - api_start + + if not xunfei_result: + raise ValueError("讯飞API返回空结果") + + # 5. 结果处理时间 + result_process_start = time.time() + xunfei_details = format_xunfei_details(xunfei_result) + xunfei_total_score = get_xunfei_total_score(xunfei_result) + timing_info['result_process'] = time.time() - result_process_start + + # 6. 数据更新时间 + update_start = time.time() + with lock: + results_dict[index] = { + '讯飞总分': xunfei_total_score, + '讯飞明细': xunfei_details + } + timing_info['data_update'] = time.time() - update_start + + # 计算总耗时 + total_time = time.time() - start_time + timing_info['total'] = total_time + + # 详细的时间分析日志 + logging.info(f"第 {index + 1} 行处理成功 - 总分: {xunfei_total_score} | " + f"总耗时: {total_time:.2f}s | " + f"速率等待: {timing_info['rate_limit_wait']:.2f}s | " + f"预处理: {timing_info['preprocess']:.3f}s | " + f"音频下载: {timing_info['audio_download']:.2f}s | " + f"API调用: {timing_info['api_call']:.2f}s | " + f"结果处理: {timing_info['result_process']:.3f}s | " + f"数据更新: {timing_info['data_update']:.3f}s") + + except Exception as api_error: + total_time = time.time() - start_time + logging.error(f"第 {index + 1} 行讯飞API调用失败: {str(api_error)} | " + f"总耗时: {total_time:.2f}s | " + f"音频下载: {timing_info.get('audio_download', 0):.2f}s | " + f"API调用: {timing_info.get('api_call', 0):.2f}s") + with lock: + results_dict[index] = { + '讯飞总分': 0, + '讯飞明细': "", + 'error': f'API调用失败: {str(api_error)}' + } + + finally: + # 7. 清理时间 + cleanup_start = time.time() + try: + if audio_file_path and os.path.exists(audio_file_path): + os.remove(audio_file_path) + logging.debug(f"已删除临时文件: {audio_file_path}") + except Exception as cleanup_error: + logging.warning(f"清理临时文件失败: {str(cleanup_error)}") + timing_info['cleanup'] = time.time() - cleanup_start + + # 释放速率限制令牌 + if rate_limiter: + try: + rate_limiter.put(None, timeout=1) # 归还令牌 + except: + pass # 队列可能已满，忽略 + + except Exception as e: + total_time = time.time() - start_time + logging.error(f"第 {index + 1} 行处理异常: {str(e)} | 总耗时: {total_time:.2f}s") + with lock: + results_dict[index] = { + '讯飞总分': 0, + '讯飞明细': "", + 'error': f'处理异常: {str(e)}' + } + + # 释放速率限制令牌 + if rate_limiter: + try: + rate_limiter.put(None, timeout=1) + except: + pass + + +def process_excel_with_xunfei_concurrent(input_file_path, output_dir="output/audio", max_workers=5, rate_limit_per_second=5): + """ + 处理Excel文件，添加讯飞评测结果（并发版本，增强控制） + + Args: + input_file_path (str): 输入Excel文件路径 + output_dir (str): 输出目录路径，默认为 output/audio + max_workers (int): 最大并发线程数，默认5 + rate_limit_per_second (int): 每秒最大请求数，默认5 + + Returns: + bool: 处理是否成功 + """ + start_time = time.time() + + try: + # 读取Excel文件 + logging.info(f"正在读取Excel文件: {input_file_path}") + df = pd.read_excel(input_file_path) + + # 检查必要的列是否存在 + required_columns = ['refText', 'userAudio'] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + logging.error(f"Excel文件缺少必要的列: {missing_columns}") + return False + + # 数据预处理和验证 + total_rows = len(df) + valid_rows = 0 + for index, row in df.iterrows(): + if pd.notna(row.get('refText')) and pd.notna(row.get('userAudio')): + valid_rows += 1 + + logging.info(f"总行数: {total_rows}, 有效行数: {valid_rows}") + + if valid_rows == 0: + logging.warning("没有找到有效的数据行") + return False + + # 添加新列 + df['讯飞总分'] = 0 + df['讯飞明细'] = "" + + # 创建优化的速率限制器 + effective_rate_limit = max(rate_limit_per_second, max_workers) + rate_limiter = Queue(maxsize=effective_rate_limit * 2) + + # 预填充令牌 + for _ in range(effective_rate_limit): + rate_limiter.put(None) + + # 启动优化的速率限制器补充线程 + def rate_limiter_refill(): + interval = 1.0 / effective_rate_limit + while True: + time.sleep(interval) + try: + rate_limiter.put(None, block=False) + except: + pass + + rate_thread = threading.Thread(target=rate_limiter_refill, daemon=True) + rate_thread.start() + + logging.info(f"速率限制设置: {effective_rate_limit} req/s (原始: {rate_limit_per_second}, 队列大小: {effective_rate_limit * 2})") + + # 创建临时目录用于下载音频文件 + with tempfile.TemporaryDirectory() as temp_dir: + logging.info(f"创建临时目录: {temp_dir}") + logging.info(f"开始并发处理，最大并发数: {max_workers}, 有效速率限制: {effective_rate_limit} req/s") + + # 准备数据 + row_data_list = [(index, row) for index, row in df.iterrows()] + + # 创建结果字典和线程锁 + results_dict = {} + lock = threading.Lock() + + # 使用线程池进行并发处理 + with ThreadPoolExecutor(max_workers=max_workers) as executor: + # 提交所有任务 + future_to_index = { + executor.submit(process_single_row, row_data, temp_dir, results_dict, lock, rate_limiter): row_data[0] + for row_data in row_data_list + } + + # 等待任务完成并显示进度 + completed_count = 0 + success_count = 0 + error_count = 0 + + for future in as_completed(future_to_index): + completed_count += 1 + index = future_to_index[future] + + try: + future.result() # 获取结果，如果有异常会抛出 + + # 检查处理结果 + with lock: + result = results_dict.get(index, {}) + if result.get('error') is None: + success_count += 1 + else: + error_count += 1 + + # 显示进度 + if completed_count % 10 == 0 or completed_count == total_rows: + elapsed_time = time.time() - start_time + avg_time_per_item = elapsed_time / completed_count + remaining_time = avg_time_per_item * (total_rows - completed_count) + + logging.info(f"进度: {completed_count}/{total_rows} ({completed_count/total_rows*100:.1f}%) " + f"成功: {success_count}, 失败: {error_count}, " + f"预计剩余时间: {remaining_time:.1f}秒") + + except Exception as e: + error_count += 1 + logging.error(f"任务 {index + 1} 执行异常: {str(e)}") + with lock: + if index not in results_dict: + results_dict[index] = { + '讯飞总分': 0, + '讯飞明细': "", + 'error': f'任务执行异常: {str(e)}' + } + + # 将结果更新到DataFrame + logging.info("正在更新结果到DataFrame...") + for index in results_dict: + result = results_dict[index] + df.at[index, '讯飞总分'] = result.get('讯飞总分', 0) + df.at[index, '讯飞明细'] = result.get('讯飞明细', "") + + # 如果有错误，可以选择记录到备注列（如果存在） + if result.get('error') and '备注' in df.columns: + existing_note = str(df.at[index, '备注']) if pd.notna(df.at[index, '备注']) else "" + error_note = f"讯飞API错误: {result['error']}" + df.at[index, '备注'] = f"{existing_note}\n{error_note}".strip() + + # 创建输出目录 + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # 生成输出文件路径 + input_path = Path(input_file_path) + output_file_path = output_path / f"{input_path.stem}_add_xunfei_result.xlsx" + + # 保存结果 + logging.info(f"正在保存结果到: {output_file_path}") + df.to_excel(output_file_path, index=False) + + # 计算总耗时 + total_time = time.time() - start_time + + # 统计处理结果 + final_success_count = sum(1 for result in results_dict.values() if result.get('error') is None) + final_error_count = len(results_dict) - final_success_count + + logging.info("=" * 50) + logging.info("并发处理完成！") + logging.info(f"处理统计: 成功 {final_success_count} 条，失败 {final_error_count} 条，总计 {len(results_dict)} 条") + logging.info(f"总耗时: {total_time:.2f} 秒") + logging.info(f"平均处理时间: {total_time/len(results_dict):.2f} 秒/条") + logging.info(f"输出文件: {output_file_path}") + logging.info("=" * 50) + + return True + + except Exception as e: + logging.error(f"处理Excel文件时出错: {str(e)}") + return False + + +def process_excel_with_xunfei(input_file_path, output_dir="output/audio"): + """ + 处理Excel文件，添加讯飞评测结果（串行版本） + + Args: + input_file_path (str): 输入Excel文件路径 + output_dir (str): 输出目录路径，默认为 output/audio + + Returns: + bool: 处理是否成功 + """ + try: + # 读取Excel文件 + print(f"正在读取Excel文件: {input_file_path}") + df = pd.read_excel(input_file_path) + + # 检查必要的列是否存在 + required_columns = ['refText', 'userAudio'] + missing_columns = [col for col in required_columns if col not in df.columns] + if missing_columns: + print(f"错误: Excel文件缺少必要的列: {missing_columns}") + return False + + # 添加新列 + df['讯飞总分'] = 0 + df['讯飞明细'] = "" + + # 创建临时目录用于下载音频文件 + with tempfile.TemporaryDirectory() as temp_dir: + print(f"创建临时目录: {temp_dir}") + + # 处理每一行数据 + total_rows = len(df) + for index, row in df.iterrows(): + print(f"\n处理进度: {index + 1}/{total_rows}") + + ref_text = str(row['refText']) if pd.notna(row['refText']) else "" + audio_url = str(row['userAudio']) if pd.notna(row['userAudio']) else "" + + if not ref_text or not audio_url: + print(f"第 {index + 1} 行数据不完整，跳过") + continue + + print(f"参考文本: {ref_text}") + print(f"音频URL: {audio_url}") + + # 下载音频文件 + audio_file_path = download_audio_file(audio_url, temp_dir) + if not audio_file_path: + print(f"第 {index + 1} 行音频下载失败，跳过") + continue + + # 调用讯飞API进行评测 + print("正在调用讯飞API进行评测...") + try: + xunfei_result = evaluate_audio_file(audio_file_path, ref_text) + print(f"讯飞API返回结果: {json.dumps(xunfei_result, indent=2, ensure_ascii=False)}") + + # 提取总分和明细 + total_score = get_xunfei_total_score(xunfei_result) + details = format_xunfei_details(xunfei_result) + + # 更新DataFrame + df.at[index, '讯飞总分'] = total_score + df.at[index, '讯飞明细'] = details + + print(f"讯飞总分: {total_score}") + print(f"讯飞明细: {details}") + + except Exception as e: + print(f"第 {index + 1} 行讯飞API调用失败: {str(e)}") + continue + + # 删除临时音频文件 + try: + os.remove(audio_file_path) + except: + pass + + # 添加延时避免API调用过于频繁 + time.sleep(1) + + # 创建输出目录 + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # 生成输出文件路径 + input_path = Path(input_file_path) + output_file_path = output_path / f"{input_path.stem}_add_xunfei_result.xlsx" + + # 保存结果 + print(f"\n正在保存结果到: {output_file_path}") + df.to_excel(output_file_path, index=False) + print("处理完成！") + + return True + + except Exception as e: + print(f"处理Excel文件时出错: {str(e)}") + return False + + +if __name__ == "__main__": + # ==================== 配置参数 ==================== + input_file = "user_audio_data_20251210_152807_sample.xlsx" + output_directory = "output/audio" # 输出目录，可以修改 + use_concurrent = True # True: 使用并发版本，False: 使用串行版本 + + # DEBUG 模式开关（True: 显示详细调试信息，False: 仅显示关键信息） + enable_debug = False # 可以设置为 True 来查看详细的 DEBUG 日志 + + # 设置全局 DEBUG_MODE + globals()['DEBUG_MODE'] = enable_debug + + # 检查环境变量 + required_env_vars = ['XUNFEI_APPID', 'XUNFEI_API_SECRET', 'XUNFEI_API_KEY'] + missing_vars = [var for var in required_env_vars if not os.environ.get(var)] + + if missing_vars: + print(f"错误: 缺少必要的环境变量: {missing_vars}") + print("请在 .env 文件或系统环境变量中配置:") + print(" XUNFEI_APPID=你的应用ID") + print(" XUNFEI_API_SECRET=你的API密钥") + print(" XUNFEI_API_KEY=你的API Key") + elif not os.path.exists(input_file): + print(f"文件不存在: {input_file}") + print("请确保Excel文件存在并包含 'refText' 和 'userAudio' 列") + else: + if use_concurrent: + print("使用并发版本处理（5路并发，5 req/s）...") + success = process_excel_with_xunfei_concurrent( + input_file, + output_dir=output_directory, + max_workers=5, + rate_limit_per_second=5 + ) + else: + print("使用串行版本处理...") + success = process_excel_with_xunfei(input_file, output_dir=output_directory) + + if success: + print("处理成功！") + else: + print("处理失败！") diff --git a/business_knowledge/git_scripts/export_component_record.py b/business_knowledge/git_scripts/export_component_record.py new file mode 100644 index 0000000..6149a19 --- /dev/null +++ b/business_knowledge/git_scripts/export_component_record.py @@ -0,0 +1,492 @@ +""" +互动组件数据导出 + +需求 20251123： +--------- +在 PGsql数据库中 筛选数据 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +支持输入时间范围 +起始时间 和 截止时间 配置格式: "20250110" + +数据表中的时间字段为 updated_at , 格式样例: "2025-11-05 19:35:46.698246+08:00" + +在这些时间范围内，筛选以下字段数据 导出为excel文件: + +c_type 与 c_id 非空 + +输出以下字段： +user_id, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +写一个简单清晰的 数据导出脚本， 输入参数都直接在脚本开头定义和修改。 不要改动文件开头的需求描述，直接追加代码。 +------- + +需求二: +读取上述 输出的 excel 文件， 围绕 每个组件进行 统计， + +统计方式如下: +仅计算 c_type 与 c_id 非空 的记录 + +以每个 c_type + c_id 拼接 后 作为统计维度， +统计以下数据: +总数量 +Perfect数量:play_result=="Perfect" 的数量 +Good数量:play_result=="Good" 的数量 +Pass数量:play_result=="Pass" 的数量 +Oops数量:play_result=="Oops" 的数量 +Failed数量:play_result=="Failed" 的数量 +Perfect+Good数量:play_result=="Perfect" 或 play_result=="Good" 的数量 +Perfect比例:Perfect数量 / 总数量 +Good比例:Good数量 / 总数量 +Pass比例:Pass数量 / 总数量 +Oops比例:Oops数量 / 总数量 +Failed比例:Failed数量 / 总数量 +Perfect+Good比例:Perfect+Good数量 / 总数量 + +导出为excel 命名: 步骤1文件 结尾追加 _stats.xlsx + +需求三: +在需求二中， 追加从另外两个mysql表关联的组件配置字段: +MYSQL_HOST=xxx +MYSQL_USERNAME=xxx +MYSQL_PASSWORD=xxx +MYSQL_DATABASE=xxx +MYSQL_PORT=xxx + +以上环境变量已配置在 .env 中。 + +1.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +增加以下字段: +title +component_config +组件类型 + +其中: + “组件类型”: 根据以下映射 把 c_type 转成中文名:xx互动 +{ + "词汇类": { + "物品互动": "mid_vocab_item", + "图片互动": "mid_vocab_image", + "填词互动": "mid_vocab_fillBlank", + "指令互动": "mid_vocab_instruction" + }, + "句子类": { + "对话互动": "mid_sentence_dialogue", + "语音互动": "mid_sentence_voice", + "材料互动": "mid_sentence_material", + "造句互动": "mid_sentence_makeSentence" + }, + "语法类": { + "挖空互动": "mid_grammar_cloze", + "组句互动": "mid_grammar_sentence" + }, + "发音类": { + "发音互动": "mid_pron_pron" + +} + +2. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +增加以下字段: +title +component_config +组件类型 + +其中: + “组件类型”: 根据以下映射 把 c_type 转成中文名:xx互动 +{ + "口语类": { + "口语快答": "core_speaking_reply", + "口语妙问": "core_speaking_inquiry", + "口语探讨": "core_speaking_explore" + "口语独白": "core_speaking_monologue" + }, + "阅读类": { + "合作阅读": "core_reading_order", + }, + "听力类": { + "合作听力": "core_listening_order", + }, + "写作类": { + "看图组句": "core_writing_imgMakeSentence", + "看图撰写": "core_writing_imgWrite", + "问题组句": "core_writing_questionMakeSentence", + "问题撰写": "core_writing_questionWrite", + }, +} + +以上追加字段 增加到 步骤二输出的表中 + + + +""" + +import os +from datetime import datetime +from dotenv import load_dotenv +import psycopg2 +import pandas as pd +import pymysql + +# ==================== 配置参数 ==================== +# 时间范围配置（格式: "20250110"） +START_DATE = "20250915" # 起始日期 +END_DATE = "20251122" # 截止日期 + +# 输出文件路径 +OUTPUT_DIR = "output" + +# 执行步骤控制 +RUN_STEP1 = False # 是否执行步骤1：数据导出 +RUN_STEP2 = True # 是否执行步骤2：数据统计 +# ================================================== + +# c_type 到中文组件类型的映射 +C_TYPE_MAPPING = { + # middle_interaction_component 映射 + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + + # core_interaction_component 映射 + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def step1_export_data(): + """步骤1：从数据库导出数据""" + print("=" * 60) + print("步骤1：数据导出") + print("=" * 60) + + # 加载环境变量 + load_dotenv() + + # 获取数据库配置 + db_config = { + 'host': os.getenv('PG_DB_HOST'), + 'port': os.getenv('PG_DB_PORT'), + 'user': os.getenv('PG_DB_USER'), + 'password': os.getenv('PG_DB_PASSWORD'), + 'database': os.getenv('PG_DB_DATABASE') + } + + # 转换时间格式 + start_datetime = datetime.strptime(START_DATE, "%Y%m%d").strftime("%Y-%m-%d 00:00:00") + end_datetime = datetime.strptime(END_DATE, "%Y%m%d").strftime("%Y-%m-%d 23:59:59") + + print(f"时间范围: {start_datetime} ~ {end_datetime}") + + # 连接数据库 + conn = psycopg2.connect(**db_config) + + # 存储所有表的数据 + all_data = [] + + # 遍历8个分表 + for i in range(8): + table_name = f"user_component_play_record_{i}" + print(f"正在读取表: {table_name}") + + # SQL查询 + query = f""" + SELECT + user_id, + session_id, + c_type, + c_id, + play_result, + user_behavior_info, + updated_at + FROM {table_name} + WHERE updated_at >= %s + AND updated_at <= %s + AND c_type IS NOT NULL + AND c_id IS NOT NULL + """ + + # 执行查询 + df = pd.read_sql_query(query, conn, params=(start_datetime, end_datetime)) + all_data.append(df) + print(f" - 读取到 {len(df)} 条记录") + + # 关闭数据库连接 + conn.close() + + # 合并所有数据 + result_df = pd.concat(all_data, ignore_index=True) + print(f"\n总共获取 {len(result_df)} 条记录") + + # 移除 updated_at 字段的时区信息（Excel不支持带时区的datetime） + if 'updated_at' in result_df.columns and not result_df.empty: + result_df['updated_at'] = result_df['updated_at'].dt.tz_localize(None) + + # 确保输出目录存在 + os.makedirs(OUTPUT_DIR, exist_ok=True) + + # 生成输出文件名 + output_filename = f"component_record_{START_DATE}_{END_DATE}.xlsx" + output_path = os.path.join(OUTPUT_DIR, output_filename) + + # 导出到Excel + result_df.to_excel(output_path, index=False, engine='openpyxl') + print(f"数据已导出到: {output_path}") + print() + + return output_path + + +def get_component_info_from_mysql(stats_df): + """从MySQL获取组件配置信息""" + # 加载环境变量 + load_dotenv() + + # 获取MySQL配置 + mysql_config = { + 'host': os.getenv('MYSQL_HOST'), + 'user': os.getenv('MYSQL_USERNAME'), + 'password': os.getenv('MYSQL_PASSWORD'), + 'database': os.getenv('MYSQL_DATABASE'), + 'port': int(os.getenv('MYSQL_PORT', 3306)), + 'charset': 'utf8mb4' + } + + print("正在连接MySQL数据库...") + conn = pymysql.connect(**mysql_config) + + try: + # 分别处理 mid 和 core 类型的组件 + mid_records = stats_df[stats_df['c_type'].str.startswith('mid', na=False)][['c_type', 'c_id']] + core_records = stats_df[stats_df['c_type'].str.startswith('core', na=False)][['c_type', 'c_id']] + + # 存储组件信息的字典，key 为 "c_type-c_id" + component_info = {} + + # 查询 middle_interaction_component 表 + if not mid_records.empty: + print(f"正在查询 middle_interaction_component 表，共 {len(mid_records)} 个组件...") + + # 获取唯一的 c_type 和 c_id 组合 + mid_unique = mid_records.drop_duplicates() + + for _, row in mid_unique.iterrows(): + c_type = row['c_type'] + c_id = row['c_id'] + + query = """ + SELECT title, component_config + FROM middle_interaction_component + WHERE c_type = %s AND c_id = %s + """ + result = pd.read_sql_query(query, conn, params=(c_type, c_id)) + + if not result.empty: + key = f"{c_type}-{c_id}" + component_info[key] = { + 'title': result['title'].iloc[0], + 'component_config': result['component_config'].iloc[0] + } + + print(f" - 查询到 {len([k for k in component_info.keys() if k.startswith('mid')])} 个组件信息") + + # 查询 core_interaction_component 表 + if not core_records.empty: + print(f"正在查询 core_interaction_component 表，共 {len(core_records)} 个组件...") + + # 获取唯一的 c_type 和 c_id 组合 + core_unique = core_records.drop_duplicates() + + for _, row in core_unique.iterrows(): + c_type = row['c_type'] + c_id = row['c_id'] + + query = """ + SELECT title, component_config + FROM core_interaction_component + WHERE c_type = %s AND c_id = %s + """ + result = pd.read_sql_query(query, conn, params=(c_type, c_id)) + + if not result.empty: + key = f"{c_type}-{c_id}" + component_info[key] = { + 'title': result['title'].iloc[0], + 'component_config': result['component_config'].iloc[0] + } + + print(f" - 查询到 {len([k for k in component_info.keys() if k.startswith('core')])} 个组件信息") + + finally: + conn.close() + + return component_info + + +def step2_statistics(input_file): + """步骤2：数据统计""" + print("=" * 60) + print("步骤2：数据统计") + print("=" * 60) + + # 读取步骤1导出的Excel文件，c_id作为字符串读取以保留前导零 + print(f"正在读取文件: {input_file}") + df = pd.read_excel(input_file, engine='openpyxl', dtype={'c_id': str}) + print(f"读取到 {len(df)} 条记录") + + # 筛选 c_type 和 c_id 非空的记录 + df_filtered = df[(df['c_type'].notna()) & (df['c_id'].notna())].copy() + print(f"筛选后 {len(df_filtered)} 条有效记录") + + # 确保c_type和c_id都是字符串类型（保留c_id的前导零） + df_filtered['c_type'] = df_filtered['c_type'].astype(str) + df_filtered['c_id'] = df_filtered['c_id'].astype(str) + + # 创建组件ID（c_type-c_id） + df_filtered['component_id'] = df_filtered['c_type'] + '-' + df_filtered['c_id'] + + # 按组件ID分组统计 + stats_list = [] + + for component_id, group in df_filtered.groupby('component_id'): + # 获取原始的 c_type 和 c_id + c_type = group['c_type'].iloc[0] + c_id = group['c_id'].iloc[0] + + # 总数量 + total_count = len(group) + + # 各状态数量 + perfect_count = len(group[group['play_result'] == 'Perfect']) + good_count = len(group[group['play_result'] == 'Good']) + pass_count = len(group[group['play_result'] == 'Pass']) + oops_count = len(group[group['play_result'] == 'Oops']) + failed_count = len(group[group['play_result'] == 'Failed']) + perfect_good_count = len(group[group['play_result'].isin(['Perfect', 'Good'])]) + + # 计算比例（保留两位小数） + perfect_ratio = round(perfect_count / total_count, 2) if total_count > 0 else 0 + good_ratio = round(good_count / total_count, 2) if total_count > 0 else 0 + pass_ratio = round(pass_count / total_count, 2) if total_count > 0 else 0 + oops_ratio = round(oops_count / total_count, 2) if total_count > 0 else 0 + failed_ratio = round(failed_count / total_count, 2) if total_count > 0 else 0 + perfect_good_ratio = round(perfect_good_count / total_count, 2) if total_count > 0 else 0 + + stats_list.append({ + 'component_id': component_id, + 'c_type': c_type, + 'c_id': c_id, + '总数量': total_count, + 'Perfect数量': perfect_count, + 'Good数量': good_count, + 'Pass数量': pass_count, + 'Oops数量': oops_count, + 'Failed数量': failed_count, + 'Perfect+Good数量': perfect_good_count, + 'Perfect比例': perfect_ratio, + 'Good比例': good_ratio, + 'Pass比例': pass_ratio, + 'Oops比例': oops_ratio, + 'Failed比例': failed_ratio, + 'Perfect+Good比例': perfect_good_ratio + }) + + # 创建统计结果DataFrame + stats_df = pd.DataFrame(stats_list) + + print(f"统计了 {len(stats_df)} 个不同的组件") + + # 从MySQL获取组件配置信息 + print("\n" + "=" * 60) + print("正在从MySQL获取组件配置信息...") + print("=" * 60) + component_info = get_component_info_from_mysql(stats_df) + + # 添加新字段：title, component_config, 组件类型 + # 使用 component_id (c_type-c_id) 作为 key 来匹配 + stats_df['title'] = stats_df['component_id'].apply(lambda x: component_info.get(x, {}).get('title', '')) + stats_df['component_config'] = stats_df['component_id'].apply(lambda x: component_info.get(x, {}).get('component_config', '')) + stats_df['组件类型'] = stats_df['c_type'].apply(lambda x: C_TYPE_MAPPING.get(x, '')) + + # 重新排列列顺序：将新增字段放在 c_type, c_id 后面 + columns_order = [ + 'component_id', 'c_type', 'c_id', + 'title', 'component_config', '组件类型', # 新增字段 + '总数量', + 'Perfect数量', 'Good数量', 'Pass数量', 'Oops数量', 'Failed数量', 'Perfect+Good数量', + 'Perfect比例', 'Good比例', 'Pass比例', 'Oops比例', 'Failed比例', 'Perfect+Good比例' + ] + stats_df = stats_df[columns_order] + + # 生成输出文件名（在原文件名后追加_stats） + output_filename = os.path.basename(input_file).replace('.xlsx', '_stats.xlsx') + output_path = os.path.join(OUTPUT_DIR, output_filename) + + # 导出到Excel + stats_df.to_excel(output_path, index=False, engine='openpyxl') + print(f"\n统计结果已导出到: {output_path}") + print() + + return output_path + + +def main(): + export_file = None + + # 执行步骤1：数据导出 + if RUN_STEP1: + export_file = step1_export_data() + + # 执行步骤2：数据统计 + if RUN_STEP2: + # 如果步骤1没有执行，需要手动指定文件路径 + if export_file is None: + export_file = os.path.join(OUTPUT_DIR, f"component_record_{START_DATE}_{END_DATE}.xlsx") + if not os.path.exists(export_file): + print(f"错误：找不到文件 {export_file}") + print("请先执行步骤1或确保文件存在") + return + + step2_statistics(export_file) + + print("=" * 60) + print("处理完成！") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/export_lesson_review.py b/business_knowledge/git_scripts/export_lesson_review.py new file mode 100644 index 0000000..8808023 --- /dev/null +++ b/business_knowledge/git_scripts/export_lesson_review.py @@ -0,0 +1,572 @@ +""" +** 不要改动我的需求描述，直接在需求后面写代码即可 ** + +课程巩固 数据导出 和 分析 + +----------- +需求一: +在 PGsql数据库中 筛选数据 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: user_unit_review_question_result + +支持输入时间范围 +起始时间 和 截止时间 配置格式: "20250110" + +数据表中的时间字段为 updated_at , 格式样例: "2025-11-05 19:35:46.698246+08:00" + +在这些时间范围内，筛选数据 (要求deleted_at字段内容为null) + +导出以下字段: + +user_id +unit_id （读取每条记录的story_id， 根据 get_id_2_unit_index 函数返回的映射表 映射到 unit_id） +lesson_id （读取chapter_id， 根据该值 查询 mysql表 vala_game_chapter 的 id == chapter_id， 并返回该记录的 index字段的值） +question_list +题目总数 +正确数量 +正确率 +play_time_seconds （读取 play_time 把ms数据转换为秒 保留整数部分） +updated_at + +其中 题目总数 正确数量 正确率 都通过 question_list 计算， +该字段为 list of json: +[ + { + "question": { + "type": "vocab_meaning_meaning", + "id": "20-0", + "title": "“clean” 的意思是什么？", + "npcId": -1 + }, + "answers": [ + "2" + ], + "optionList": [ + { + "option": "爬行" + }, + { + "option": "清晰的" + }, + { + "option": "清洁" + } + ], + "isRight": true + }, + ... +] + +每个元素为一道题目， 题目中有 "isRight": true 代表用户做对了。 + +导出为excel文件 +---- +需求二 基于 需求一的输出文件 作为 输入文件 进行数据聚合。 + +聚合的维度是每道题目 + +根据 question_list 中的 每个题目 取 question -> id 作为唯一标识 + +统计每个题目 +总记录数量 +正确数量 +正确率 + +并查询mysql表 补充题目的以下信息: +步骤一中，每个题目id的格式是 num1-num2 (question -> id) +查询vala_kp_question表 +其中num1部分 用于 检索vala_kp_question 中的 id, 每个id下 可能有多道题目 在 vala_kp_question的 question 字段 是一个list, num2为question 字段中的索引 + +补充以下字段: +kp_id (vala_kp_question字段) +category (vala_kp_question字段) +skill (vala_kp_question字段) +type (vala_kp_question字段) +题目配置 (question字段中 对应 num2 索引的内容) + +最终针对每道题目输出以下字段: +出现位置 (list, 把所有出现的位置拼接 unit_id +"_"+ lesson_id 例如:"unit10-lesson1" 这样的格式) +question_id (question -> id) +kp_id (vala_kp_question字段) +category (vala_kp_question字段) +skill (vala_kp_question字段) +type (vala_kp_question字段) +题目配置 (question字段中 对应 num2 索引的内容) +总记录数量 +正确数量 +正确率 + +导出为excel 命名为 步骤一文件_stat.xlsx + +所有需要配置的参数 放在脚本开头位置 + +""" + +import os +import pymysql +import psycopg2 +from psycopg2.extras import RealDictCursor +from datetime import datetime +import pandas as pd +from dotenv import load_dotenv +import json +from collections import defaultdict + +# 加载环境变量 +load_dotenv() + +# ============ 配置参数 ============ +START_DATE = "20250915" # 起始时间 +END_DATE = "20251122" # 截止时间 +OUTPUT_NAME = "lesson_review_data_{}_{}.xlsx".format(START_DATE, END_DATE) # 输出文件名 +OUTPUT_FILENAME = os.path.join("./output", OUTPUT_NAME) +# ================================= + +def get_mysql_connection(): + """获取MySQL连接""" + db_host = os.getenv('MYSQL_HOST') + db_user = os.getenv('MYSQL_USERNAME') + db_password = os.getenv('MYSQL_PASSWORD') + db_name = os.getenv('MYSQL_DATABASE') + db_port = os.getenv('MYSQL_PORT') + + if not all([db_host, db_user, db_password, db_name]): + raise Exception("Error: Missing MySQL configuration in .env file.") + + connection = pymysql.connect( + host=db_host, + user=db_user, + password=db_password, + database=db_name, + port=int(db_port) if db_port else 3306, + cursorclass=pymysql.cursors.DictCursor + ) + return connection + +def get_pgsql_connection(): + """获取PGsql连接""" + pg_host = os.getenv('PG_DB_HOST') + pg_port = os.getenv('PG_DB_PORT') + pg_user = os.getenv('PG_DB_USER') + pg_password = os.getenv('PG_DB_PASSWORD') + pg_database = os.getenv('PG_DB_DATABASE') + + if not all([pg_host, pg_port, pg_user, pg_password, pg_database]): + raise Exception("Error: Missing PGsql configuration in .env file.") + + connection = psycopg2.connect( + host=pg_host, + port=int(pg_port), + user=pg_user, + password=pg_password, + database=pg_database, + cursor_factory=RealDictCursor + ) + return connection + +def get_id_2_unit_index(): + """获取story_id到unit_id的映射""" + print("正在获取 story_id 到 unit_id 的映射...") + connection = get_mysql_connection() + + try: + with connection.cursor() as cursor: + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + cursor.execute(sql) + results = cursor.fetchall() + + id_2_unit_index = {} + for index, row in enumerate(results): + id_2_unit_index[row['id']] = index + + print(f"成功获取 {len(id_2_unit_index)} 个单元映射") + return id_2_unit_index + finally: + connection.close() + +def get_chapter_id_to_lesson_id(): + """获取chapter_id到lesson_id的映射""" + print("正在获取 chapter_id 到 lesson_id 的映射...") + connection = get_mysql_connection() + + try: + with connection.cursor() as cursor: + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + cursor.execute(sql) + results = cursor.fetchall() + + chapter_id_to_lesson_id = {} + for row in results: + chapter_id_to_lesson_id[row['id']] = row['index'] + + print(f"成功获取 {len(chapter_id_to_lesson_id)} 个课程映射") + return chapter_id_to_lesson_id + finally: + connection.close() + +def analyze_question_list(question_list_json): + """分析题目列表，返回题目总数、正确数量、正确率""" + try: + if isinstance(question_list_json, str): + question_list = json.loads(question_list_json) + else: + question_list = question_list_json + + if not isinstance(question_list, list): + return 0, 0, 0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0 + + return total, correct, accuracy + except Exception as e: + print(f"解析题目列表出错: {e}") + return 0, 0, 0 + +def export_step1(): + """需求一：导出原始数据""" + print("=" * 50) + print("开始执行需求一：导出原始数据") + print("=" * 50) + + # 获取映射关系 + id_2_unit_index = get_id_2_unit_index() + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id() + + # 连接PGsql + print("正在连接 PGsql 数据库...") + pg_conn = get_pgsql_connection() + + try: + with pg_conn.cursor() as cursor: + # 构建时间范围 + start_datetime = datetime.strptime(START_DATE, "%Y%m%d") + end_datetime = datetime.strptime(END_DATE, "%Y%m%d") + end_datetime = end_datetime.replace(hour=23, minute=59, second=59) + + sql = """ + SELECT user_id, story_id, chapter_id, question_list, play_time, updated_at + FROM user_unit_review_question_result + WHERE updated_at >= %s + AND updated_at <= %s + AND deleted_at IS NULL + ORDER BY updated_at + """ + + print(f"查询时间范围: {start_datetime} 至 {end_datetime}") + cursor.execute(sql, (start_datetime, end_datetime)) + results = cursor.fetchall() + + print(f"查询到 {len(results)} 条记录") + + # 处理数据 + export_data = [] + for row in results: + user_id = row['user_id'] + story_id = row['story_id'] + chapter_id = row['chapter_id'] + question_list_raw = row['question_list'] + play_time = row['play_time'] + updated_at = row['updated_at'] + + # 确保 question_list 是 Python 对象（PGsql 的 jsonb 会自动转换） + # 如果是字符串，先解析；如果已经是对象，直接使用 + if isinstance(question_list_raw, str): + try: + question_list = json.loads(question_list_raw) + except: + question_list = [] + else: + question_list = question_list_raw if question_list_raw else [] + + # 映射 unit_id + unit_id = id_2_unit_index.get(story_id, -1) + + # 映射 lesson_id + lesson_id = chapter_id_to_lesson_id.get(chapter_id, -1) + + # 分析题目列表 + total, correct, accuracy = analyze_question_list(question_list) + + # 转换播放时长（ms -> s） + play_time_seconds = int(play_time / 1000) if play_time else 0 + + # 转换question_list为字符串（统一序列化为JSON字符串） + question_list_str = json.dumps(question_list, ensure_ascii=False) if question_list else "" + + # 移除时区信息（Excel不支持带时区的datetime） + updated_at_no_tz = updated_at.replace(tzinfo=None) if updated_at else None + + export_data.append({ + 'user_id': user_id, + 'unit_id': unit_id, + 'lesson_id': lesson_id, + 'question_list': question_list_str, + '题目总数': total, + '正确数量': correct, + '正确率': accuracy, + 'play_time_seconds': play_time_seconds, + 'updated_at': updated_at_no_tz + }) + + # 导出到Excel + df = pd.DataFrame(export_data) + + # 确保输出目录存在 + os.makedirs(os.path.dirname(OUTPUT_FILENAME), exist_ok=True) + + df.to_excel(OUTPUT_FILENAME, index=False, engine='openpyxl') + print(f"成功导出 {len(export_data)} 条记录到: {OUTPUT_FILENAME}") + + return OUTPUT_FILENAME + + finally: + pg_conn.close() + +def get_all_kp_questions(question_ids): + """批量获取所有题目信息，避免N+1查询问题""" + print(f"正在批量查询 {len(question_ids)} 道题目的信息...") + + # 解析所有question_id，获取需要查询的kp_question id列表 + kp_ids = set() + for qid in question_ids: + try: + parts = qid.split('-') + if len(parts) == 2: + kp_ids.add(int(parts[0])) + except: + continue + + print(f"需要查询 {len(kp_ids)} 条 vala_kp_question 记录") + + # 批量查询MySQL + connection = get_mysql_connection() + kp_data_map = {} + + try: + with connection.cursor() as cursor: + # 使用IN查询批量获取 + if kp_ids: + placeholders = ','.join(['%s'] * len(kp_ids)) + sql = f""" + SELECT id, kp_id, category, skill, type, question + FROM vala_kp_question + WHERE id IN ({placeholders}) AND deleted_at IS NULL + """ + cursor.execute(sql, tuple(kp_ids)) + results = cursor.fetchall() + + print(f"成功查询到 {len(results)} 条记录") + + # 构建映射表 + for row in results: + kp_data_map[row['id']] = row + finally: + connection.close() + + # 为每个question_id构建结果 + question_info_map = {} + for question_id in question_ids: + try: + parts = question_id.split('-') + if len(parts) != 2: + question_info_map[question_id] = (None, None, None, None, None) + continue + + kp_id = int(parts[0]) + question_index = int(parts[1]) + + kp_data = kp_data_map.get(kp_id) + if not kp_data: + question_info_map[question_id] = (None, None, None, None, None) + continue + + # 解析question字段 + question_list = kp_data['question'] + if isinstance(question_list, str): + question_list = json.loads(question_list) + + # 获取指定索引的题目配置 + question_config = None + if isinstance(question_list, list) and 0 <= question_index < len(question_list): + question_config = json.dumps(question_list[question_index], ensure_ascii=False) + + question_info_map[question_id] = ( + kp_data['kp_id'], + kp_data['category'], + kp_data['skill'], + kp_data['type'], + question_config + ) + except Exception as e: + print(f"处理题目信息出错 ({question_id}): {e}") + question_info_map[question_id] = (None, None, None, None, None) + + return question_info_map + +def export_step2(input_filename): + """需求二：数据聚合统计""" + print("=" * 50) + print("开始执行需求二：数据聚合统计") + print("=" * 50) + + # 读取步骤一的输出文件 + print(f"正在读取文件: {input_filename}") + df = pd.read_excel(input_filename, engine='openpyxl') + + print(f"读取到 {len(df)} 条记录") + + # 按题目聚合统计 + question_stats = defaultdict(lambda: { + 'locations': set(), + 'total_count': 0, + 'correct_count': 0 + }) + + parse_success_count = 0 + parse_fail_count = 0 + empty_question_list_count = 0 + processed_question_count = 0 + + for idx, row in df.iterrows(): + unit_id = row['unit_id'] + lesson_id = row['lesson_id'] + question_list_str = row['question_list'] + + # 解析question_list + try: + if pd.isna(question_list_str) or not question_list_str: + question_list = [] + empty_question_list_count += 1 + else: + question_list = json.loads(question_list_str) + parse_success_count += 1 + except Exception as e: + question_list = [] + parse_fail_count += 1 + if parse_fail_count <= 3: + print(f"[警告] 第 {idx+1} 条记录解析失败: {e}") + + # 统计每道题目 + for question_item in question_list: + if not isinstance(question_item, dict): + continue + + question = question_item.get('question', {}) + question_id = question.get('id') + is_right = question_item.get('isRight', False) + + if not question_id: + continue + + # 添加出现位置 + location = f"unit{unit_id}-lesson{lesson_id}" + question_stats[question_id]['locations'].add(location) + + # 统计数量 + question_stats[question_id]['total_count'] += 1 + if is_right: + question_stats[question_id]['correct_count'] += 1 + + processed_question_count += 1 + + print(f"\n解析统计:") + print(f" - 解析成功: {parse_success_count} 条") + print(f" - 解析失败: {parse_fail_count} 条") + print(f" - question_list 为空: {empty_question_list_count} 条") + print(f" - 处理的题目总数: {processed_question_count} 道") + print(f" - 聚合得到不同题目: {len(question_stats)} 道") + + # 批量获取所有题目信息（优化性能） + all_question_ids = list(question_stats.keys()) + question_info_map = get_all_kp_questions(all_question_ids) + + # 构建导出数据 + print(f"\n正在构建导出数据...") + export_data = [] + for idx, (question_id, stats) in enumerate(question_stats.items()): + if (idx + 1) % 100 == 0: + print(f" 已处理 {idx + 1}/{len(question_stats)} 道题目") + + # 从批量查询结果中获取题目信息 + kp_id, category, skill, type_field, question_config = question_info_map.get( + question_id, (None, None, None, None, None) + ) + + # 计算正确率 + total = stats['total_count'] + correct = stats['correct_count'] + accuracy = round(correct / total * 100, 2) if total > 0 else 0 + + # 出现位置列表 + locations_list = sorted(list(stats['locations'])) + locations_str = ', '.join(locations_list) + + export_data.append({ + '出现位置': locations_str, + 'question_id': question_id, + 'kp_id': kp_id, + 'category': category, + 'skill': skill, + 'type': type_field, + '题目配置': question_config, + '总记录数量': total, + '正确数量': correct, + '正确率': accuracy + }) + + # 导出到Excel + output_stat_filename = input_filename.replace('.xlsx', '_stat.xlsx') + df_stat = pd.DataFrame(export_data) + + print(f"\n正在导出到 Excel...") + df_stat.to_excel(output_stat_filename, index=False, engine='openpyxl') + + print(f"成功导出 {len(export_data)} 道题目的统计数据到: {output_stat_filename}") + + return output_stat_filename + +def main(): + """主函数""" + try: + # 执行需求一 + step1_output = export_step1() + + print("\n") + + # 执行需求二 + step2_output = export_step2(step1_output) + + print("\n" + "=" * 50) + print("所有任务完成！") + print(f"需求一输出文件: {step1_output}") + print(f"需求二输出文件: {step2_output}") + print("=" * 50) + + except Exception as e: + print(f"执行出错: {e}") + import traceback + traceback.print_exc() + +if __name__ == "__main__": + main() + + + diff --git a/business_knowledge/git_scripts/export_mid_config.py b/business_knowledge/git_scripts/export_mid_config.py new file mode 100644 index 0000000..c536621 --- /dev/null +++ b/business_knowledge/git_scripts/export_mid_config.py @@ -0,0 +1,181 @@ +""" +MYSQL_HOST=xxx +MYSQL_USERNAME=xxx +MYSQL_PASSWORD=xxx +MYSQL_DATABASE=xxx +MYSQL_PORT=xxx + +以上环境变量已配置在 .env 中。 + +我要导出一个数据表的某些记录 并添加一些字段。 + +表名:middle_interaction_component + +根据 c_id 过滤数据: +c_id为 7位 字符串 其中 {两位季度编号}{两位单元编号}{三位组件编号} 过滤其中 单元编号部分为 00~20 以及 26 的对应记录 也就是 xx00xxx ~ xx20xxx 以及 xx26xxx 的记录 + +导出以下字段: +id +c_type +c_id +title +component_config +related_path +kp_relation_info +created_at +updated_at + +新增以下字段: +1. “组件类型”: 根据以下映射 把 c_type 转成中文名:xx互动 +{ + "词汇类": { + "物品互动": "mid_vocab_item", + "图片互动": "mid_vocab_image", + "填词互动": "mid_vocab_fillBlank", + "指令互动": "mid_vocab_instruction" + }, + "句子类": { + "对话互动": "mid_sentence_dialogue", + "语音互动": "mid_sentence_voice", + "材料互动": "mid_sentence_material", + "造句互动": "mid_sentence_makeSentence" + }, + "语法类": { + "挖空互动": "mid_grammar_cloze", + "组句互动": "mid_grammar_sentence" + }, + "发音类": { + "发音互动": "mid_pron_pron" + +} + +2. “是否关联了知识点”: 如果 kp_relation_info 不为空 且包含至少一个具体的知识点编号 则为 “是” 否则为 “否” +有效关联知识点的一个样例数据:[{"kpId":"0326011","kpType":"sentence","kpTitle":"What does... look like?","kpSkill":"sentence_meaning","kpSkillName":"语义"}] + +3. "是否已组课"： 如果 related_path 不为空 则为 “是” 否则为 “否” +一个有效的 related_path 样例: {"packageId":13,"unitId":40,"lessonId":213,"packageIndex":3,"unitIndex":2,"lessonIndex":2} + +4. “前置对话”: +component_config 中的 preDialog 字段， 如果不存在 则为 “空” +{"asrPrompt":"","cId":"0326022","cType":"mid_sentence_dialogue","meaning":"语义；语音","mode":"read","postDialog":[{"content":"Leave it to me.","npcId":540,"npcName":"Victoria","type":"npc"}],"preDialog":[{"content":"But do we still have time?","npcId":30,"type":"user"}],"question":{"content":"What if we miss the spaceship?","mode":"read","npcId":30,"type":"user"},"resourceMapping":{"Medic":503},"title":"询问万一错过飞船怎么办"} + +5. "后置对话": +component_config 中的 postDialog 字段， 如果不存在 则为 “空” + +6. 前置/后置对话中非user角色数量 +component_config 中的 preDialog 以及 postDialog 字段中， 统计所有 type 为 npc ,根据 npcId 去重后的角色数量 +例如 +--- +前置对话： +[{"content":"But do we still have time?","npcId":30,"type":"user"}] +后置对话： +[{"content":"Leave it to me.","npcId":540,"npcName":"Victoria","type":"npc"}] +非user角色数量： 1 +--- + +--- +前置对话： +[{"content":"But do we still have time?","npcId":31,"type":"npc","npcName":"Ben"}] +后置对话： +[{"content":"Leave it to me.","npcId":540,"npcName":"Victoria","type":"npc"}] +非user角色数量： 2 +--- + +最终输出一个 excel文档。 + +""" + +import os +import json +from datetime import datetime +import pymysql +import pandas as pd +from dotenv import load_dotenv + +load_dotenv() + +# 组件类型映射 +TYPE_MAP = { + "mid_vocab_item": "物品互动", "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动" +} + +def get_data(): + conn = pymysql.connect( + host=os.getenv('MYSQL_HOST'), port=int(os.getenv('MYSQL_PORT', 3306)), + user=os.getenv('MYSQL_USERNAME'), password=os.getenv('MYSQL_PASSWORD'), + database=os.getenv('MYSQL_DATABASE'), charset='utf8mb4' + ) + + # 构建c_id过滤条件 + conditions = [f"c_id LIKE '__{i:02d}___'" for i in range(21)] + ["c_id LIKE '__26___'"] + where_clause = " OR ".join(conditions) + + sql = f"""SELECT id, c_type, c_id, title, component_config, related_path, + kp_relation_info, created_at, updated_at + FROM middle_interaction_component WHERE {where_clause}""" + + df = pd.read_sql(sql, conn) + conn.close() + return df + +def process_data(df): + # 组件类型 + df['组件类型'] = df['c_type'].map(TYPE_MAP).fillna(df['c_type']) + + # 是否关联知识点 + def check_kp(kp_info): + if not kp_info: return "否" + try: + data = json.loads(kp_info) + return "是" if isinstance(data, list) and any(item.get('kpId') for item in data) else "否" + except: return "否" + + df['是否关联了知识点'] = df['kp_relation_info'].apply(check_kp) + + # 是否已组课 + def check_lesson(path): + if not path: return "否" + try: return "是" if json.loads(path) else "否" + except: return "否" + + df['是否已组课'] = df['related_path'].apply(check_lesson) + + # 前置/后置对话及NPC统计 + def extract_dialog(config, dialog_type): + if not config: return "空" + try: + data = json.loads(config) + dialog = data.get(dialog_type, []) + return json.dumps(dialog, ensure_ascii=False) if dialog else "空" + except: return "空" + + def count_npc(config): + if not config: return 0 + try: + data = json.loads(config) + npc_ids = set() + for dialog in ['preDialog', 'postDialog']: + for item in data.get(dialog, []): + if item.get('type') == 'npc' and 'npcId' in item: + npc_ids.add(item['npcId']) + return len(npc_ids) + except: return 0 + + df['前置对话'] = df['component_config'].apply(lambda x: extract_dialog(x, 'preDialog')) + df['后置对话'] = df['component_config'].apply(lambda x: extract_dialog(x, 'postDialog')) + df['前置/后置对话中非user角色数量'] = df['component_config'].apply(count_npc) + + return df + +if __name__ == "__main__": + df = get_data() + df = process_data(df) + + filename = f"middle_interaction_component_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx" + df.to_excel(filename, index=False) + print(f"导出完成: {filename}") diff --git a/business_knowledge/git_scripts/export_realtime_asr.py b/business_knowledge/git_scripts/export_realtime_asr.py new file mode 100644 index 0000000..e042530 --- /dev/null +++ b/business_knowledge/git_scripts/export_realtime_asr.py @@ -0,0 +1,385 @@ +""" +导出 流式语音音频 脚本 + +v1.0 +--- +原始数据存储于ES数据库中 +索引: llm_realtime_asr_log + +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx （注意这里可能有特殊符号） + +需要配置的内容放置在脚本最开头 +开始时间 (8位数字年月日) +截止时间 (8位数字年月日) + +仅筛选 时间范围内的数据记录 +可以基于 timestamp_int 字段内容进行时间筛选 格式样例:1,769,496,892 + +正常情况 每个 voice_id 会对应两条记录 +可以 以 voice_id为单位 +最终 按照每个 voice_id 聚合出以下数据: + +asr_prompt （其中一条记录会有这个内容） +result_str （其中一条记录会有这个内容） +timestamp (两条记录都会有，保留最新的一条对应的时间) 格式样例: 2023-12-12 12:12:12 +voice_id +audio_url 按以下规则拼接: https://static.valavala.com/vala_llm/realtime_asr_audio_backup/online/{8位年月日}/{voice_id}.wav 8位年月日 基于 timestamp计算 格式 20260121这种 +source （其中一条记录会有这个内容） + +最终导出一个excel。 +--- + +""" + +import os +from datetime import datetime +import requests +import pandas as pd +from dotenv import load_dotenv +from collections import defaultdict +import urllib3 + +# ==================== 配置区域 ==================== +START_DATE = "20251201" # 开始日期 (8位数字年月日) +END_DATE = "20260131" # 结束日期 (8位数字年月日) +# ================================================= + +# 加载环境变量 +load_dotenv() + +# ES配置 +ES_HOST = os.getenv("ES_HOST") +ES_PORT = int(os.getenv("ES_PORT", "9200")) +ES_SCHEME = os.getenv("ES_SCHEME", "https") +ES_USER = os.getenv("ES_USER", "elastic") +ES_PASSWORD = os.getenv("ES_PASSWORD") +ES_INDEX = "llm_realtime_asr_log" + +# 每批处理的数据量 +SCROLL_SIZE = 1000 +SCROLL_TIMEOUT = "5m" + + +def timestamp_int_from_date(date_str): + """将8位日期字符串转换为timestamp_int（秒级时间戳）""" + dt = datetime.strptime(date_str, "%Y%m%d") + return int(dt.timestamp()) + + +def format_timestamp(ts): + """将时间戳转换为格式化字符串""" + if isinstance(ts, (int, float)): + return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") + return ts + + +def generate_audio_url(voice_id, timestamp): + """生成audio_url""" + date_str = datetime.fromtimestamp(timestamp).strftime("%Y%m%d") + return f"https://static.valavala.com/vala_llm/realtime_asr_audio_backup/online/{date_str}/{voice_id}.wav" + + +def connect_es(): + """测试ES连接""" + print("正在测试 Elasticsearch 连接...") + + # 禁用SSL警告 + if ES_SCHEME == "https": + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + + base_url = f"{ES_SCHEME}://{ES_HOST}:{ES_PORT}" + auth = (ES_USER, ES_PASSWORD) if ES_USER and ES_PASSWORD else None + + try: + # 测试连接 + resp = requests.get( + base_url, + auth=auth, + timeout=10, + verify=False if ES_SCHEME == "https" else True + ) + resp.raise_for_status() + + print(f"✓ 成功连接到 Elasticsearch: {ES_HOST}:{ES_PORT}") + return True + except Exception as e: + print(f"✗ 连接失败: {e}") + return False + + +def query_data(start_date, end_date): + """查询ES数据""" + start_ts = timestamp_int_from_date(start_date) + end_ts = timestamp_int_from_date(end_date) + 86400 # 结束日期加一天，包含当天数据 + + print(f"\n开始查询数据...") + print(f"时间范围: {start_date} 至 {end_date}") + print(f"时间戳范围: {start_ts} 至 {end_ts}") + + # 禁用SSL警告 + if ES_SCHEME == "https": + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + + base_url = f"{ES_SCHEME}://{ES_HOST}:{ES_PORT}" + search_url = f"{base_url}/{ES_INDEX}/_search" + headers = {"Content-Type": "application/json"} + auth = (ES_USER, ES_PASSWORD) if ES_USER and ES_PASSWORD else None + + query = { + "query": { + "range": { + "timestamp_int": { + "gte": start_ts, + "lt": end_ts + } + } + }, + "sort": [{"timestamp_int": {"order": "asc"}}], + "size": SCROLL_SIZE + } + + try: + # 初始查询（使用scroll） + params = {"scroll": SCROLL_TIMEOUT} + response = requests.post( + search_url, + headers=headers, + json=query, + auth=auth, + params=params, + timeout=30, + verify=False if ES_SCHEME == "https" else True + ) + response.raise_for_status() + data = response.json() + + scroll_id = data.get("_scroll_id") + total_hits = data["hits"]["total"]["value"] + + print(f"✓ 查询完成，共找到 {total_hits} 条记录") + + return data, scroll_id, total_hits + + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + +def aggregate_by_voice_id(response, scroll_id, total_hits): + """按voice_id聚合数据""" + voice_data = defaultdict(list) + processed_count = 0 + + print("\n开始处理数据...") + + # 禁用SSL警告 + if ES_SCHEME == "https": + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + + base_url = f"{ES_SCHEME}://{ES_HOST}:{ES_PORT}" + scroll_url = f"{base_url}/_search/scroll" + headers = {"Content-Type": "application/json"} + auth = (ES_USER, ES_PASSWORD) if ES_USER and ES_PASSWORD else None + + while True: + hits = response["hits"]["hits"] + + if not hits: + break + + for hit in hits: + source = hit["_source"] + voice_id = source.get("voice_id") + + if voice_id: + voice_data[voice_id].append(source) + + processed_count += 1 + + # 打印进度 + progress = (processed_count / total_hits) * 100 + print(f"\r处理进度: {processed_count}/{total_hits} ({progress:.1f}%)", end="") + + # 获取下一批数据 + try: + scroll_response = requests.post( + scroll_url, + headers=headers, + json={ + "scroll": SCROLL_TIMEOUT, + "scroll_id": scroll_id + }, + auth=auth, + timeout=30, + verify=False if ES_SCHEME == "https" else True + ) + scroll_response.raise_for_status() + response = scroll_response.json() + + # 更新 scroll_id（可能会变化） + scroll_id = response.get("_scroll_id", scroll_id) + + except Exception as e: + print(f"\n✗ 获取下一批数据失败: {e}") + break + + print(f"\n✓ 数据处理完成，共处理 {processed_count} 条记录") + print(f"✓ 找到 {len(voice_data)} 个唯一的 voice_id") + + # 清理scroll + try: + clear_scroll_url = f"{base_url}/_search/scroll" + requests.delete( + clear_scroll_url, + headers=headers, + json={"scroll_id": [scroll_id]}, + auth=auth, + timeout=10, + verify=False if ES_SCHEME == "https" else True + ) + except Exception: + pass # 清理失败不影响结果 + + return voice_data + + +def merge_voice_records(voice_data): + """合并voice_id的记录，只保留恰好2条记录的""" + print("\n开始聚合 voice_id 数据...") + + merged_data = [] + valid_count = 0 + invalid_count = 0 + + for voice_id, records in voice_data.items(): + # 只处理恰好有2条记录的voice_id + if len(records) != 2: + invalid_count += 1 + continue + + valid_count += 1 + + # 初始化合并后的数据 + merged_record = { + "voice_id": voice_id, + "asr_prompt": None, + "result_str": None, + "timestamp": None, + "source": None, + "audio_url": None + } + + # 找出最新的timestamp + max_timestamp = max( + records[0].get("timestamp_int", 0), + records[1].get("timestamp_int", 0) + ) + + # 合并数据 + for record in records: + if record.get("asr_prompt"): + merged_record["asr_prompt"] = record["asr_prompt"] + if record.get("result_str"): + merged_record["result_str"] = record["result_str"] + if record.get("source"): + merged_record["source"] = record["source"] + + # 设置timestamp和audio_url + merged_record["timestamp"] = format_timestamp(max_timestamp) + merged_record["audio_url"] = generate_audio_url(voice_id, max_timestamp) + + merged_data.append(merged_record) + + print(f"✓ 聚合完成") + print(f" - 有效记录（2条/voice_id）: {valid_count}") + print(f" - 无效记录（非2条/voice_id）: {invalid_count}") + + return merged_data + + +def export_to_excel(data, start_date, end_date): + """导出到Excel""" + if not data: + print("\n警告: 没有数据可导出") + return + + print(f"\n开始导出数据到 Excel...") + + # 创建DataFrame + df = pd.DataFrame(data) + + # 调整列顺序 + columns = ["voice_id", "asr_prompt", "result_str", "timestamp", "audio_url", "source"] + df = df[columns] + + # 生成文件名 + output_dir = "output" + os.makedirs(output_dir, exist_ok=True) + filename = f"realtime_asr_export_{start_date}_{end_date}.xlsx" + filepath = os.path.join(output_dir, filename) + + # 导出Excel + df.to_excel(filepath, index=False, engine="openpyxl") + + print(f"✓ 数据已导出到: {filepath}") + print(f"✓ 共导出 {len(df)} 条记录") + + +def main(): + """主函数""" + print("=" * 60) + print("流式语音 ASR 数据导出工具 v1.0") + print("=" * 60) + + start_time = datetime.now() + + try: + # 测试ES连接 + if not connect_es(): + raise Exception("无法连接到 Elasticsearch，请检查配置") + + # 查询数据 + response, scroll_id, total_hits = query_data(START_DATE, END_DATE) + + if total_hits == 0: + print("\n没有找到符合条件的数据") + return + + # 聚合数据 + voice_data = aggregate_by_voice_id(response, scroll_id, total_hits) + + # 合并记录 + merged_data = merge_voice_records(voice_data) + + # 导出Excel + export_to_excel(merged_data, START_DATE, END_DATE) + + # 统计耗时 + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + + print(f"\n{'=' * 60}") + print(f"✓ 任务完成! 总耗时: {duration:.2f} 秒") + print(f"{'=' * 60}") + + except Exception as e: + print(f"\n✗ 错误: {str(e)}") + import traceback + traceback.print_exc() + + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/export_resource_name.py b/business_knowledge/git_scripts/export_resource_name.py new file mode 100644 index 0000000..36506d6 --- /dev/null +++ b/business_knowledge/git_scripts/export_resource_name.py @@ -0,0 +1,121 @@ +""" +MYSQL_HOST=xxx +MYSQL_USERNAME=xxx +MYSQL_PASSWORD=xxx +MYSQL_DATABASE=xxx +MYSQL_PORT=xxx + +以上环境变量已配置在 .env 中。 + +我要导出一个数据表的某些记录 并添加一些字段。 + +表名:vala_resource_base + +过滤全部 type == "角色" 的记录 + +导出以下字段: +id +cn_name +en_name + + +最终输出到 excel文档。 "角色资源导出_251031.xlsx" + +""" + +import os +import pandas as pd +import pymysql +from dotenv import load_dotenv +from datetime import datetime + +def load_config(): + """加载环境变量配置""" + load_dotenv() + + config = { + 'host': os.getenv('MYSQL_HOST'), + 'user': os.getenv('MYSQL_USERNAME'), + 'password': os.getenv('MYSQL_PASSWORD'), + 'database': os.getenv('MYSQL_DATABASE'), + 'port': int(os.getenv('MYSQL_PORT', 3306)), + 'charset': 'utf8mb4' + } + + # 验证配置 + for key, value in config.items(): + if value is None and key != 'charset': + raise ValueError(f"环境变量 {key} 未配置") + + return config + +def connect_mysql(config): + """连接MySQL数据库""" + try: + connection = pymysql.connect(**config) + print("MySQL数据库连接成功") + return connection + except Exception as e: + print(f"MySQL数据库连接失败: {e}") + raise + +def export_role_resources(): + """导出角色资源数据""" + try: + # 加载配置 + config = load_config() + + # 连接数据库 + connection = connect_mysql(config) + + # SQL查询语句 + sql = """ + SELECT + id, + cn_name, + en_name + FROM vala_resource_base + WHERE type = '角色' + ORDER BY id + """ + + print("开始查询数据...") + + # 执行查询并获取数据 + df = pd.read_sql(sql, connection) + + print(f"查询到 {len(df)} 条记录") + + # 关闭数据库连接 + connection.close() + + # 导出到Excel文件 + output_filename = "角色资源导出_251031.xlsx" + df.to_excel(output_filename, index=False, engine='openpyxl') + + print(f"数据已成功导出到: {output_filename}") + print(f"导出字段: {list(df.columns)}") + print(f"导出记录数: {len(df)}") + + # 显示前几行数据预览 + if len(df) > 0: + print("\n数据预览:") + print(df.head()) + + return output_filename + + except Exception as e: + print(f"导出过程中发生错误: {e}") + raise + +if __name__ == "__main__": + try: + print("开始导出角色资源数据...") + print(f"执行时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + + output_file = export_role_resources() + + print(f"\n✅ 导出完成! 文件保存为: {output_file}") + + except Exception as e: + print(f"\n❌ 导出失败: {e}") diff --git a/business_knowledge/git_scripts/export_unit_challenge_data.py b/business_knowledge/git_scripts/export_unit_challenge_data.py new file mode 100644 index 0000000..9bfedd4 --- /dev/null +++ b/business_knowledge/git_scripts/export_unit_challenge_data.py @@ -0,0 +1,343 @@ +""" +** 不要改动我的需求描述，直接在需求后面写代码即可 ** + +需求一: +先写一个最简单脚本 实现下面sql功能 + +SELECT * FROM `vala_game_info` WHERE id > 0 AND `vala_game_info`.`deleted_at` IS NULL ORDER BY season_package_id asc,`index` asc + +环境变量读取: +MYSQL_HOST=xxx +MYSQL_USERNAME=xxx +MYSQL_PASSWORD=xxx +MYSQL_DATABASE=xxx +MYSQL_PORT=xxx +----------- +需求二: +在 PGsql数据库中 筛选数据 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表:user_unit_challenge_question_result + +支持输入时间范围 +起始时间 和 截止时间 配置格式: "20250110" + +数据表中的时间字段为 updated_at , 格式样例: "2025-11-05 19:35:46.698246+08:00" + +在这些时间范围内，筛选数据 (要求deleted_at字段内容为null) + +导出以下字段: + +user_id +unit_id （读取每条记录的story_id， 根据 get_id_2_unit_index 函数返回的映射表 映射到 unit_id） +score_text +question_list +updated_at +category +play_time_seconds （读取 play_time 把ms数据转换为秒 保留整数部分） + +导出为excel文件 + +配置参数直接在脚本开头给出即可 + +需求三: +需求二中 作为步骤一 +本需求为步骤二 基于 步骤一的 文档 +进行数据聚合 + +根据每个unit_id + category 进行分组 + +统计每个分组下的以下数值: +总记录数量 +Perfect数量 (读取 score_text =="Perfect") +Good数量 (读取 score_text =="Good") +Oops数量 (读取 score_text =="Oops") +Perfect率 (Perfect数量 / 总记录数量) +Good率 (Good数量 / 总记录数量) +Oops率 (Oops数量 / 总记录数量) + +导出为excel 命名为 步骤一名字_stats.xlsx + +""" + +import os +import pymysql +import psycopg2 +from psycopg2.extras import RealDictCursor +from datetime import datetime +import pandas as pd +from dotenv import load_dotenv + +# 加载环境变量 +load_dotenv() + +# ============ 配置参数 ============ +START_DATE = "20250915" # 起始时间 +END_DATE = "20251128" # 截止时间 +OUTPUT_NAME = "unit_challenge_data_{}_{}.xlsx".format(START_DATE, END_DATE) # 输出文件名 +OUTPUT_FILENAME = os.path.join("./output", OUTPUT_NAME) +# ================================= + +def get_id_2_unit_index(): + # 读取数据库配置 + db_host = os.getenv('MYSQL_HOST') + db_user = os.getenv('MYSQL_USERNAME') + db_password = os.getenv('MYSQL_PASSWORD') + db_name = os.getenv('MYSQL_DATABASE') + db_port = os.getenv('MYSQL_PORT') + + # 简单的参数检查 + if not all([db_host, db_user, db_password, db_name]): + print("Error: Missing database configuration in .env file.") + print("Ensure MYSQL_HOST, MYSQL_USERNAME, MYSQL_PASSWORD, MYSQL_DATABASE are set.") + return + + try: + # 连接数据库 + connection = pymysql.connect( + host=db_host, + user=db_user, + password=db_password, + database=db_name, + port=int(db_port) if db_port else 3306, + cursorclass=pymysql.cursors.DictCursor + ) + + print(f"Connected to database: {db_host}") + + try: + with connection.cursor() as cursor: + # 定义 SQL 语句 + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + + print(f"Executing SQL: {sql}") + + # 执行查询 + cursor.execute(sql) + + # 获取所有结果 + results = cursor.fetchall() + + print(f"Total records found: {len(results)}") + print("-" * 30) + + # 打印结果 + print(results) + id_2_unit_index = {} + for index, row in enumerate(results): + id_2_unit_index[row['id']] = index + + print("映射结果:") + print(id_2_unit_index) + + + + print("-" * 30) + print("Done.") + return id_2_unit_index + + finally: + connection.close() + + except Exception as e: + print(f"An error occurred: {e}") + + +def export_unit_challenge_data(start_date, end_date, output_filename): + """ + 从PostgreSQL数据库导出单元挑战数据 + """ + # 读取PostgreSQL数据库配置 + pg_host = os.getenv('PG_DB_HOST') + pg_port = os.getenv('PG_DB_PORT') + pg_user = os.getenv('PG_DB_USER') + pg_password = os.getenv('PG_DB_PASSWORD') + pg_database = os.getenv('PG_DB_DATABASE') + + # 检查配置 + if not all([pg_host, pg_port, pg_user, pg_password, pg_database]): + print("Error: Missing PostgreSQL database configuration in .env file.") + print("Ensure PG_DB_HOST, PG_DB_PORT, PG_DB_USER, PG_DB_PASSWORD, PG_DB_DATABASE are set.") + return + + # 获取 id 到 unit_index 的映射 + print("正在获取 unit_id 映射表...") + id_2_unit_index = get_id_2_unit_index() + if not id_2_unit_index: + print("Error: Failed to get id_2_unit_index mapping.") + return + + # 转换时间格式: "20250110" -> "2025-01-10 00:00:00" + start_datetime = datetime.strptime(start_date, "%Y%m%d").strftime("%Y-%m-%d 00:00:00") + end_datetime = datetime.strptime(end_date, "%Y%m%d").strftime("%Y-%m-%d 00:00:00") + + print(f"时间范围: {start_datetime} 至 {end_datetime}") + + try: + # 连接PostgreSQL数据库 + connection = psycopg2.connect( + host=pg_host, + port=int(pg_port), + user=pg_user, + password=pg_password, + database=pg_database, + cursor_factory=RealDictCursor + ) + + print(f"已连接到 PostgreSQL 数据库: {pg_host}") + + try: + with connection.cursor() as cursor: + # 定义SQL查询 + sql = """ + SELECT + user_id, + story_id, + score_text, + question_list, + updated_at, + category, + play_time + FROM user_unit_challenge_question_result + WHERE deleted_at IS NULL + AND updated_at >= %s + AND updated_at < %s + ORDER BY updated_at ASC + """ + + print(f"执行查询...") + + # 执行查询 + cursor.execute(sql, (start_datetime, end_datetime)) + + # 获取所有结果 + results = cursor.fetchall() + + print(f"查询到 {len(results)} 条记录") + + # 处理数据 + export_data = [] + for row in results: + # 映射 story_id 到 unit_id + story_id = row['story_id'] + unit_id = id_2_unit_index.get(story_id, None) + + # 转换 play_time (毫秒) 为秒 (整数) + play_time_seconds = row['play_time'] // 1000 if row['play_time'] else 0 + + # 移除 updated_at 的时区信息（Excel 不支持带时区的 datetime） + updated_at = row['updated_at'] + if updated_at and hasattr(updated_at, 'replace'): + updated_at = updated_at.replace(tzinfo=None) + + export_data.append({ + 'user_id': row['user_id'], + 'unit_id': unit_id, + 'score_text': row['score_text'], + 'question_list': row['question_list'], + 'updated_at': updated_at, + 'category': row['category'], + 'play_time_seconds': play_time_seconds + }) + + # 导出到Excel + if export_data: + df = pd.DataFrame(export_data) + df.to_excel(output_filename, index=False, engine='openpyxl') + print(f"数据已导出到: {output_filename}") + print(f"共导出 {len(export_data)} 条记录") + else: + print("没有数据可导出") + + finally: + connection.close() + print("数据库连接已关闭") + + except Exception as e: + print(f"发生错误: {e}") + + +def aggregate_stats(input_filename): + """ + 基于步骤一的Excel文件进行数据聚合 + 按 unit_id + category 分组，统计各项指标 + """ + try: + # 读取步骤一导出的Excel文件 + print(f"正在读取文件: {input_filename}") + df = pd.read_excel(input_filename, engine='openpyxl') + + print(f"读取到 {len(df)} 条记录") + + # 按 unit_id + category 分组统计 + grouped = df.groupby(['unit_id', 'category'], dropna=False) + + stats_data = [] + for (unit_id, category), group in grouped: + total_count = len(group) + perfect_count = (group['score_text'] == 'Perfect').sum() + good_count = (group['score_text'] == 'Good').sum() + oops_count = (group['score_text'] == 'Oops').sum() + + # 计算占比 + perfect_rate = round(perfect_count / total_count if total_count > 0 else 0, 2) + good_rate = round(good_count / total_count if total_count > 0 else 0, 2) + oops_rate = round(oops_count / total_count if total_count > 0 else 0, 2) + + stats_data.append({ + 'unit_id': unit_id, + 'category': category, + '总记录数量': total_count, + 'Perfect数量': perfect_count, + 'Good数量': good_count, + 'Oops数量': oops_count, + 'Perfect率': perfect_rate, + 'Good率': good_rate, + 'Oops率': oops_rate + }) + + # 生成输出文件名 + base_name = os.path.splitext(input_filename)[0] + output_filename = f"{base_name}_stats.xlsx" + + # 导出统计结果 + if stats_data: + stats_df = pd.DataFrame(stats_data) + stats_df.to_excel(output_filename, index=False, engine='openpyxl') + print(f"统计数据已导出到: {output_filename}") + print(f"共 {len(stats_data)} 个分组") + else: + print("没有数据可统计") + + except Exception as e: + print(f"数据聚合时发生错误: {e}") + + +if __name__ == "__main__": + # 步骤一：执行导出 + print("=" * 50) + print("步骤一：导出原始数据") + print("=" * 50) + export_unit_challenge_data(START_DATE, END_DATE, OUTPUT_FILENAME) + + # 步骤二：数据聚合 + print("\n" + "=" * 50) + print("步骤二：数据聚合统计") + print("=" * 50) + aggregate_stats(OUTPUT_FILENAME) + + print("\n" + "=" * 50) + print("全部完成！") + print("=" * 50) + diff --git a/business_knowledge/git_scripts/export_user_id_data.py b/business_knowledge/git_scripts/export_user_id_data.py new file mode 100644 index 0000000..ba0ddcc --- /dev/null +++ b/business_knowledge/git_scripts/export_user_id_data.py @@ -0,0 +1,1846 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = None # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = [2148] # 5095[7232] # [1783,5375,5371,5345,5303,5293,5095,4289,4494,4473,4460,4452,4386,4388,4236,4043,2758,2841,2756,2750,2692,1781,1693,2256,2234,2373] # 账户ID列表，示例：[100, 101, 102] + +OUTPUT_DIR = "output/260126/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + cur.execute( + f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """, + (user_id,), + ) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/extract_core_speaking_data.py b/business_knowledge/git_scripts/extract_core_speaking_data.py new file mode 100644 index 0000000..237d266 --- /dev/null +++ b/business_knowledge/git_scripts/extract_core_speaking_data.py @@ -0,0 +1,681 @@ +""" +筛选 整合 线上的 口语 核心互动 对话记录数据 + +数据筛选流程如下: +一 步骤一 +首先， 在 PGsql数据库中 筛选出 口语核心互动对应的 session_id. +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +支持输入时间范围 +起始时间 和 截止时间 配置格式: "20250110" + +数据表中的时间字段为 updated_at , 格式样例: "2025-11-05 19:35:46.698246+08:00" + +在这些时间范围内，筛选以下数据: +c_type 为 core_speaking_reply 或者 core_speaking_inquiry 的数据 + +输出总的数据条数 + +然后导出 中间 excel文件 + +包含以下字段： +user_id, +session_id, +c_type, +c_id, +play_result, +updated_at + +二. 步骤二 +根据 c_type 和 c_id 筛选核心互动的配置 补充一些字段。 + +需要读取配置表: +mysql表 core_interaction_component +相关环境变量在.env: +MYSQL_HOST=xxx +MYSQL_USERNAME=xxx +MYSQL_PASSWORD=xxx +MYSQL_DATABASE=xxx +MYSQL_PORT=xxx + +基于 c_type 和 c_id 字段匹配， 在 步骤一表格内容基础上追加以下字段: +title +reference_dialog 从 component_config 中抽取出 reference_dialog 字段的内容。 +component_config内容样例: +``` +{"taskInfo":{"cId":"0000001","cType":"core_speaking_inquiry","title":"询问种植甜瓜的信息","taskDesc":"向Ben提问甜瓜种植的最佳季节、浇水频率和成熟的季节；","sceneDesc":"我和Ben到甜味城，参观了水果资源站和种植园。Ben的妈妈Kate讲了种植知识，我们都很感兴趣，想一起种甜瓜。我不懂，便问Ben，他虽没种过、不确定，还是告诉我注意事项。","img":"","key":[{"desc":"询问种植信息","keyList":[{"type":"default","npcId":269,"content":"Have you ever planted a ...?","desc":"你种过......吗？"},{"type":"default","npcId":269,"content":"What season is the best time to plant ...?","desc":"种植......的最佳时间是哪个季节？"},{"type":"default","npcId":269,"content":"Do ... need ... every day?","desc":"......需要每天浇......么？"}]}]},"dialogSetting":{"setting":{"npcName":"Ben","npcId":287,"round":5,"checkRound":3}},"dialogConfig":{"config":{"asrPrompt":"melon,summer,autumn,water,frequency,plant,season,harvest","promptInfo":{"default":"# 1. 角色（你要扮演谁）\n- 你是 Ben，一个 8 岁的小男孩，对种植水果感兴趣但不太确定具体细节。\n- 语言风格：简单、直接，偶尔带有不确定的语气。\n- 示例表达：\n - \"I think summer. It's warm then.\"\n - \"Maybe every two days? Not every day, I think.\"\n\n# 2. 任务（你如何参与到整个对话）\n- 你需要只在用户提问时提供信息，不会主动提及种植甜瓜的具体细节。\n- 如果用户提问相关内容，你需要根据知识库中的信息回答，不编造或偏离。\n- 如果用户的问题不清晰，你需要尝试澄清后再作答。\n- 如果用户长时间不提问或偏离主题，你需要温和、自然地进行交谈，引导回到主题。\n- 当所有知识点已传达后，你需要鼓励用户开始行动。\n\n# 3. 背景信息（引用配置）\n`你是 Ben，你和用户来到了甜味城Sweet Town。你们参观了水果资源站和种植园。在种植园中，你的妈妈Kate给你和用户介绍了一些种植水果的知识。你和用户对此很感兴趣。你们想要一起种一颗甜瓜。用户不知道种植甜瓜的知识，于是向你提问。虽然你没有种过甜瓜，对什么都不确定。但你还是回答了用户的问题，告诉用户关于种植甜瓜需要注意的事情。`\n\n# 4. 知识库（你知道的信息）\n- 种甜瓜的最佳季节：应该在夏天\n- 种甜瓜的浇水频率：应该隔一天浇一次水\n- 甜瓜成熟的季节：秋天\n- 如果用户提问相关内容，你会用这些信息来回答。\n\n# 5. 语言风格（固定内容）\n 1. 使用标准、正式的英语，水平为 CEFR A1/A2，每句话不超过 10 个单词\n 2. 始终保持礼貌和友好\n 3. 尽量避免重复表达，适当变换措辞\n\n# 6. 开场白\n你由你开始对话，你会说：“Let's plant a melon now! Or do you still have some questions？”\n\n# 7. 回应方式（固定内容）\n`你只在用户提问时才根据知识库中的信息回答。其他时候，以符合你身份的方式，自然地进行交谈。不主动提供信息，不偏离语境。`","final_goal":"Ben 说出了种植种甜瓜的最佳季节应该在夏天、种甜瓜的浇水频率应该隔一天浇一次水以及甜瓜成熟的季节是秋天","in_progress_goal":"Ben 说出了种植种甜瓜的最佳季节应该在夏天、种甜瓜的浇水频率应该隔一天浇一次水以及甜瓜成熟的季节是秋天","reference_dialog":"# 示例对话\nBen: Let's plant a melon now! Or do you still have some questions？\nYou: Great! When is the best time to plant it?\nBen: I think summer. It's warm then. Mom said melons like warm weather.\nYou: Oh, good. How often should we water it?\nBen: Maybe every two days? Not every day, I think.\nYou: And when will it be ready to eat?\nBen: Autumn, I guess. Plant in summer, get melons in autumn. That sounds right.","scene":"#任务背景\n你是 Ben，你和用户来到了甜味城Sweet Town。你们参观了水果资源站和种植园。在种植园中，你的妈妈Kate给你和用户介绍了一些种植水果的知识。你和用户对此很感兴趣。你们想要一起种一颗甜瓜。用户不知道种植甜瓜的知识，于是向你提问。虽然你没有种过甜瓜，对什么都不确定。但你还是回答了用户的问题，告诉用户关于种植甜瓜需要注意的事情。","user_knowledge":"# 知识\n- 询问种植信息\nHave you ever planted a ...? 你种过......吗？\nWhat season is the best time to plant ...? 种植......的最佳时间是哪个季节？\nDo ... need ... every day? ......需要每天浇......么？","user_scene":"我和Ben到甜味城，参观了水果资源站和种植园。Ben的妈妈Kate讲了种植知识，我们都很感兴趣，想一起种甜瓜。我不懂，便问Ben，他虽没种过、不确定，还是告诉我注意事项。","user_task":"向Ben提问甜瓜种植的最佳季节、浇水频率和成熟的季节；"}}},"studyInfo":{"learningPart":{"learning":[{"question":{"desc":"现在你需要询问Ben关于种植甜瓜的最佳季节。"},"optionList":[{"option":"When is the best time to plant it?","feedbackDesc":"太棒了！你正确地询问了种植甜瓜的最佳季节。请大声朗读这句话！"},{"option":"How often should we water it?","feedbackDesc":"这句话是询问浇水频率的，不是询问最佳种植季节的。请再试一次，询问Ben种植甜瓜的最佳季节。"},{"option":"When will it be ready to eat?","feedbackDesc":"这句话是询问甜瓜成熟季节的，不是询问最佳种植季节的。请再试一次，询问Ben种植甜瓜的最佳季节。"}],"answer":[0],"read":{"type":"user","npcId":30,"content":"When is the best time to plant it?"},"feedback":{"type":"npc","npcName":"Ben","npcId":287,"content":"I think summer. It's warm then. Mom said melons like warm weather."}},{"question":{"desc":"Ben告诉你种植甜瓜的最佳季节是夏天。现在你需要询问Ben关于种植甜瓜的浇水频率。"},"optionList":[{"option":"When is the best time to plant it?","feedbackDesc":"这句话是询问最佳种植季节的，不是询问浇水频率的。请再试一次，询问Ben种植甜瓜的浇水频率。"},{"option":"How often should we water it?","feedbackDesc":"太棒了！你正确地询问了种植甜瓜的浇水频率。请大声朗读这句话！"},{"option":"When will it be ready to eat?","feedbackDesc":"这句话是询问甜瓜成熟季节的，不是询问浇水频率的。请再试一次，询问Ben种植甜瓜的浇水频率。"}],"answer":[1],"read":{"type":"user","npcId":30,"content":"How often should we water it?"},"feedback":{"type":"npc","npcName":"Ben","npcId":287,"content":"Maybe every two days? Not every day, I think."}},{"question":{"desc":"Ben告诉你种植甜瓜的浇水频率是隔一天一次。现在你需要询问Ben关于甜瓜成熟的季节。"},"optionList":[{"option":"When is the best time to plant it?","feedbackDesc":"这句话是询问最佳种植季节的，不是询问甜瓜成熟季节的。请再试一次，询问Ben甜瓜成熟的季节。"},{"option":"How often should we water it?","feedbackDesc":"这句话是询问浇水频率的，不是询问甜瓜成熟季节的。请再试一次，询问Ben甜瓜成熟的季节。"},{"option":"When will it be ready to eat?","feedbackDesc":"太棒了！你正确地询问了甜瓜成熟的季节。请大声朗读这句话！"}],"answer":[2],"read":{"type":"user","npcId":30,"content":"When will it be ready to eat?"},"feedback":{"type":"npc","npcName":"Ben","npcId":287,"content":"Autumn, I guess. Plant in summer, get melons in autumn. That sounds right."}}],"opening":{"type":"npc","npcName":"Ben","npcId":287,"content":"Let's plant a melon now! Or do you still have some questions？","desc":"Ben邀请你一起种植甜瓜，并询问你是否还有问题。"},"closing":{"desc":"Ben已经回答了所有关于种植甜瓜的问题，任务成功完成！"}}},"kpInfoList":[{"kpId":"","kpType":"sentence","kpTitle":"What do you think about the fight?","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"","kpType":"sentence","kpTitle":"What do you think about the fight?","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"","kpType":"sentence","kpTitle":"Can you help us?","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"","kpType":"sentence","kpTitle":"Can you help us?","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"","kpType":"sentence","kpTitle":"Do you know any way to beat him?","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"","kpType":"sentence","kpTitle":"Do you know any way to beat him?","kpSkill":"sentence_meaning","kpSkillName":"语义"}]} +``` + +追加后，excel文件包含以下字段: +user_id, +session_id, +c_type, +c_id, +play_result, +updated_at, +title +reference_dialog + +三. 步骤三 追加对话历史数据 +对话历史数据，需要根据以下es数据库来补充: + +es索引: llm_roleplayagent_round_log +相关环境变量在.env: +ES_HOST=xxx +ES_PORT=xxx +ES_SCHEME=xxx +ES_USER=xxx +ES_PASSWORD=xxx + +基于每条记录中的 session_id, 匹配 es日志中 session_id 相同 且 action为 get_chat 对应的记录，整理后， 追加为 chat_log 字段。 + +es中的日志是每轮作为一条记录，按以下逻辑进行拼接: +读取 current_round, +current round 为 0 ， 则 chat_log中加入 npc_message 的内容 "npc: " + npc_message +current round 为 1~n 按顺序 依次追加 user_input 和 npc_message ， 每轮之间用换行符隔开。 +完全拼接后 最为 chat_log 内容 +完整样例: +``` +npc:xxx +user:xxx +npc:xxx +... ... +``` + +拼接完成后 追加 chat_log 和 round_num （取最大的current_round） + +最终输出的 excel文件字段: +user_id, +session_id, +c_type, +c_id, +play_result, +updated_at, +title, +reference_dialog, +chat_log, +user_behavior_info, +round_num + + +---------------------- +根据以上需求 提供一个数据处理的脚本 尽量用高效的匹配。 我只需要输出最终的匹配文件，一个简单的功能脚本。 脚本不需要太复杂。但在输出的节点增加必要的日志 方便我了解数据量和进度 输入 时间范围 在 脚本开头配置即可。 +---------------------- + +补充需求: +pg sql数据库中 增加字段 user_behavior_info 读取。 并保留到最终的输出excel文档中 在 chat_log字段之后。 其他不变。 +---------------------- + +补充需求 25.11.07: +从 mysql表中 额外读取两个字段的信息进行处理， + +1. lesson +抽取related_path字段中的lessonIndex内容 (4): +{"packageId":2,"unitId":26,"lessonId":128,"packageIndex":1,"unitIndex":12,"lessonIndex":4} + +2. knowledge_points +直接读取 kp_relation_info 的内容。 + +3. in_progress_goal +读取 和 reference_dialog 平级的 in_progress_goal 字段内容。 + +4. final_goal +读取 和 reference_dialog 平级的 final_goal 字段内容。 + +以上四个字段 都追加到最终输出的表中， +全部输出字段顺序如下: + +user_id, +session_id, +c_type, +c_id, +play_result, +updated_at, +title, +lesson, +knowledge_points, +in_progress_goal, +final_goal, +reference_dialog, +chat_log, +user_behavior_info, +round_num + + +""" + +import os +import json +import pandas as pd +import psycopg2 +import pymysql +from elasticsearch import Elasticsearch +from datetime import datetime +from dotenv import load_dotenv +import logging + +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# 时间范围配置 - 修改这里的日期范围 +START_DATE = "20251001" # 起始时间 格式: "20250110" +END_DATE = "20251031" # 截止时间 格式: "20250131" + +class CoreSpeakingDataProcessor: + def __init__(self): + # 加载环境变量 + load_dotenv() + + # PG数据库配置 + self.pg_config = { + 'host': os.getenv('PG_DB_HOST'), + 'port': int(os.getenv('PG_DB_PORT', 5432)), + 'user': os.getenv('PG_DB_USER'), + 'password': os.getenv('PG_DB_PASSWORD'), + 'database': os.getenv('PG_DB_DATABASE') + } + + # MySQL数据库配置 + self.mysql_config = { + 'host': os.getenv('MYSQL_HOST'), + 'port': int(os.getenv('MYSQL_PORT', 3306)), + 'user': os.getenv('MYSQL_USERNAME'), + 'password': os.getenv('MYSQL_PASSWORD'), + 'database': os.getenv('MYSQL_DATABASE'), + 'charset': 'utf8mb4' + } + + # ES配置 + self.es_config = { + 'host': os.getenv('ES_HOST'), + 'port': int(os.getenv('ES_PORT', 9200)), + 'scheme': os.getenv('ES_SCHEME', 'http'), + 'user': os.getenv('ES_USER'), + 'password': os.getenv('ES_PASSWORD') + } + + self.data = None + + def convert_date_format(self, date_str): + """将'20250110'格式转换为数据库查询用的格式""" + try: + dt = datetime.strptime(date_str, '%Y%m%d') + return dt.strftime('%Y-%m-%d') + except ValueError: + logger.error(f"日期格式错误: {date_str}, 应为'20250110'格式") + raise + + def get_next_day(self, date_str): + """获取下一天的日期""" + try: + dt = datetime.strptime(date_str, '%Y%m%d') + next_day = dt + pd.Timedelta(days=1) + return next_day.strftime('%Y-%m-%d') + except ValueError: + logger.error(f"日期格式错误: {date_str}, 应为'20250110'格式") + raise + + def step1_extract_from_pg(self): + """步骤一: 从PG数据库筛选核心互动数据""" + logger.info("步骤一: 开始从PG数据库筛选数据...") + + start_date = self.convert_date_format(START_DATE) + end_date_next = self.get_next_day(END_DATE) # 获取结束日期的下一天 + logger.info(f"时间范围: {start_date} 到 {end_date_next} (不含)") + + # 构建查询SQL - 查询8个分表 + all_data = [] + table_names = [f"user_component_play_record_{i}" for i in range(8)] + + for table_name in table_names: + logger.info(f"正在处理表: {table_name}") + + # 为每个表创建独立的连接，避免事务问题 + try: + conn = psycopg2.connect(**self.pg_config) + logger.debug(f"为表 {table_name} 创建数据库连接") + except Exception as e: + logger.error(f"为表 {table_name} 创建数据库连接失败: {e}") + continue + + # 检查当前表是否存在 user_behavior_info 字段 + has_behavior_info = False + try: + with conn.cursor() as cur: + cur.execute( + """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = %s + AND column_name = 'user_behavior_info' + ) + """, + (table_name,) + ) + res = cur.fetchone() + has_behavior_info = bool(res[0]) if res else False + logger.debug(f"表 {table_name} 是否包含 user_behavior_info: {has_behavior_info}") + except Exception as e: + logger.warning(f"检测表 {table_name} 的 user_behavior_info 字段失败: {e}") + + # 动态构建查询列 + extra_col = ", user_behavior_info" if has_behavior_info else "" + sql = f""" + SELECT + user_id, + session_id, + c_type, + c_id, + play_result, + updated_at{extra_col} + FROM {table_name} + WHERE + updated_at >= %s + AND updated_at < %s + AND c_type IN ('core_speaking_reply', 'core_speaking_inquiry') + ORDER BY updated_at + """ + + try: + df = pd.read_sql(sql, conn, params=[start_date, end_date_next]) + # 保证列存在，即使部分分表没有该字段 + if 'user_behavior_info' not in df.columns: + df['user_behavior_info'] = '' + if not df.empty: + logger.info(f"表 {table_name} 获取到 {len(df)} 条数据") + all_data.append(df) + else: + logger.info(f"表 {table_name} 无符合条件的数据") + except Exception as e: + logger.error(f"查询表 {table_name} 失败: {e}") + finally: + conn.close() + + if all_data: + self.data = pd.concat(all_data, ignore_index=True) + logger.info(f"步骤一完成: 总共获取到 {len(self.data)} 条数据") + + # 统计 user_behavior_info 非空条数 + if 'user_behavior_info' in self.data.columns: + non_empty_behavior = (self.data['user_behavior_info'].astype(str).str.strip() != '').sum() + logger.info(f"步骤一: user_behavior_info 字段有值 {non_empty_behavior}/{len(self.data)} 条") + + # 处理datetime字段，去掉时区信息（Excel不支持带时区的datetime） + if 'updated_at' in self.data.columns: + self.data['updated_at'] = pd.to_datetime(self.data['updated_at']).dt.tz_localize(None) + logger.info("已处理updated_at字段的时区信息") + + # 输出中间Excel文件 + intermediate_file = f"core_speaking_step1_{START_DATE}_{END_DATE}.xlsx" + self.data.to_excel(intermediate_file, index=False) + logger.info(f"步骤一中间文件已保存: {intermediate_file}") + else: + logger.warning("步骤一: 未获取到任何数据") + self.data = pd.DataFrame() + + def step2_add_title_from_mysql(self): + """步骤二: 从MySQL补充title字段，并从component_config中提取reference_dialog等字段""" + if self.data.empty: + logger.warning("步骤二: 数据为空，跳过") + return + + logger.info("步骤二: 开始从MySQL补充title字段...") + + # 连接MySQL数据库 + try: + conn = pymysql.connect(**self.mysql_config) + logger.info("MySQL数据库连接成功") + except Exception as e: + logger.error(f"MySQL数据库连接失败: {e}") + raise + + # 获取所有需要查询的c_type和c_id组合 + unique_components = self.data[['c_type', 'c_id']].drop_duplicates() + logger.info(f"需要查询 {len(unique_components)} 个不同的组件配置") + + # 查询title、component_config、related_path和kp_relation_info + sql = """ + SELECT c_type, c_id, title, component_config, related_path, kp_relation_info + FROM core_interaction_component + WHERE (c_type, c_id) IN ({}) + """.format(','.join(['(%s,%s)'] * len(unique_components))) + + params = [] + for _, row in unique_components.iterrows(): + params.extend([row['c_type'], row['c_id']]) + + try: + title_df = pd.read_sql(sql, conn, params=params) + logger.info(f"从MySQL获取到 {len(title_df)} 条组件配置") + except Exception as e: + logger.error(f"查询MySQL失败: {e}") + title_df = pd.DataFrame(columns=['c_type', 'c_id', 'title', 'component_config', 'related_path', 'kp_relation_info']) + + conn.close() + + # 从related_path中解析lesson（lessonIndex） + def extract_lesson(related_path_str): + if related_path_str is None or related_path_str == '': + return '' + try: + data = json.loads(related_path_str) + if isinstance(data, dict): + lesson_index = data.get('lessonIndex') + return str(lesson_index) if lesson_index is not None else '' + return '' + except Exception: + return '' + + # 从component_config中解析reference_dialog、in_progress_goal和final_goal + def extract_config_fields(cfg_str): + result = { + 'reference_dialog': '', + 'in_progress_goal': '', + 'final_goal': '' + } + if cfg_str is None or cfg_str == '': + return result + try: + data = json.loads(cfg_str) + if isinstance(data, dict): + dialog_config = data.get('dialogConfig') or data.get('dialog_config') + if isinstance(dialog_config, dict): + config_obj = dialog_config.get('config') + if isinstance(config_obj, dict): + promptInfo = config_obj.get('promptInfo') + if isinstance(promptInfo, dict): + ref = promptInfo.get('reference_dialog') + result['reference_dialog'] = ref if isinstance(ref, str) else '' + + in_prog = promptInfo.get('in_progress_goal') + result['in_progress_goal'] = in_prog if isinstance(in_prog, str) else '' + + final = promptInfo.get('final_goal') + result['final_goal'] = final if isinstance(final, str) else '' + + return result + + # 兜底：如果顶层就有这些字段 + ref = data.get('reference_dialog') + result['reference_dialog'] = ref if isinstance(ref, str) else '' + + in_prog = data.get('in_progress_goal') + result['in_progress_goal'] = in_prog if isinstance(in_prog, str) else '' + + final = data.get('final_goal') + result['final_goal'] = final if isinstance(final, str) else '' + + return result + except Exception: + return result + + # 解析lesson + if 'related_path' in title_df.columns: + title_df['lesson'] = title_df['related_path'].apply(extract_lesson) + else: + title_df['lesson'] = '' + + # 解析knowledge_points（直接读取kp_relation_info） + if 'kp_relation_info' in title_df.columns: + title_df['knowledge_points'] = title_df['kp_relation_info'].fillna('') + else: + title_df['knowledge_points'] = '' + + # 解析component_config中的多个字段 + if 'component_config' in title_df.columns: + config_fields = title_df['component_config'].apply(extract_config_fields) + title_df['reference_dialog'] = config_fields.apply(lambda x: x['reference_dialog']) + title_df['in_progress_goal'] = config_fields.apply(lambda x: x['in_progress_goal']) + title_df['final_goal'] = config_fields.apply(lambda x: x['final_goal']) + else: + title_df['reference_dialog'] = '' + title_df['in_progress_goal'] = '' + title_df['final_goal'] = '' + + # 仅保留需要合并的列 + title_df = title_df[['c_type', 'c_id', 'title', 'lesson', 'knowledge_points', + 'in_progress_goal', 'final_goal', 'reference_dialog']] + + # 合并数据 + self.data = pd.merge( + self.data, + title_df, + on=['c_type', 'c_id'], + how='left' + ) + + # 填充空值 + self.data['title'] = self.data['title'].fillna('') + self.data['lesson'] = self.data['lesson'].fillna('') + self.data['knowledge_points'] = self.data['knowledge_points'].fillna('') + self.data['in_progress_goal'] = self.data['in_progress_goal'].fillna('') + self.data['final_goal'] = self.data['final_goal'].fillna('') + self.data['reference_dialog'] = self.data['reference_dialog'].fillna('') + + # 统计解析成功的字段条数 + non_empty_ref = (self.data['reference_dialog'] != '').sum() + non_empty_lesson = (self.data['lesson'] != '').sum() + non_empty_kp = (self.data['knowledge_points'] != '').sum() + non_empty_in_prog = (self.data['in_progress_goal'] != '').sum() + non_empty_final = (self.data['final_goal'] != '').sum() + + logger.info(f"步骤二完成: 已补充字段统计:") + logger.info(f" - lesson: {non_empty_lesson}/{len(self.data)} 条有值") + logger.info(f" - knowledge_points: {non_empty_kp}/{len(self.data)} 条有值") + logger.info(f" - in_progress_goal: {non_empty_in_prog}/{len(self.data)} 条有值") + logger.info(f" - final_goal: {non_empty_final}/{len(self.data)} 条有值") + logger.info(f" - reference_dialog: {non_empty_ref}/{len(self.data)} 条有值") + + # 输出中间Excel文件 + intermediate_file = f"core_speaking_step2_{START_DATE}_{END_DATE}.xlsx" + # 处理datetime字段，去掉时区信息（Excel不支持带时区的datetime） + if 'updated_at' in self.data.columns: + self.data['updated_at'] = pd.to_datetime(self.data['updated_at']).dt.tz_localize(None) + self.data.to_excel(intermediate_file, index=False) + logger.info(f"步骤二中间文件已保存: {intermediate_file}") + + def step3_add_chat_log_from_es(self): + """步骤三: 从ES补充对话历史数据""" + if self.data.empty: + logger.warning("步骤三: 数据为空，跳过") + return + + logger.info("步骤三: 开始从ES补充对话历史数据...") + + # 连接ES + try: + es_url = f"{self.es_config['scheme']}://{self.es_config['host']}:{self.es_config['port']}" + if self.es_config['user'] and self.es_config['password']: + es = Elasticsearch( + [es_url], + http_auth=(self.es_config['user'], self.es_config['password']) + ) + else: + es = Elasticsearch([es_url]) + + # 测试连接 + if es.ping(): + logger.info("ES连接成功") + else: + raise Exception("ES连接失败") + except Exception as e: + logger.error(f"ES连接失败: {e}") + # 添加空的chat_log和round_num字段 + self.data['chat_log'] = '' + self.data['round_num'] = 0 + return + + # 获取唯一的session_id + unique_sessions = self.data['session_id'].unique() + logger.info(f"需要查询 {len(unique_sessions)} 个不同的session") + + # 批量查询ES + chat_logs = {} + round_nums = {} + + batch_size = 100 + for i in range(0, len(unique_sessions), batch_size): + batch_sessions = unique_sessions[i:i+batch_size] + logger.info(f"正在处理session批次 {i//batch_size + 1}/{(len(unique_sessions)-1)//batch_size + 1}") + + try: + # 构建ES查询 + query = { + "query": { + "bool": { + "must": [ + {"terms": {"session_id": batch_sessions.tolist()}}, + {"term": {"action": "get_chat"}} + ] + } + }, + "size": 10000, + "sort": [ + {"session_id": {"order": "asc"}}, + {"current_round": {"order": "asc"}} + ] + } + + response = es.search(index="llm_roleplayagent_round_log", body=query) + hits = response['hits']['hits'] + + logger.info(f"本批次从ES获取到 {len(hits)} 条对话记录") + + # 按session_id分组处理 + session_rounds = {} + for hit in hits: + source = hit['_source'] + session_id = source.get('session_id') + current_round = source.get('current_round', 0) + + if session_id not in session_rounds: + session_rounds[session_id] = [] + + session_rounds[session_id].append({ + 'current_round': current_round, + 'user_input': source.get('user_input', ''), + 'npc_message': source.get('npc_message', '') + }) + + # 为每个session构建chat_log + for session_id, rounds in session_rounds.items(): + # 按round排序 + rounds.sort(key=lambda x: x['current_round']) + + chat_parts = [] + max_round = 0 + + for round_data in rounds: + current_round = round_data['current_round'] + max_round = max(max_round, current_round) + + if current_round == 0: + # round 0 只添加npc_message + if round_data['npc_message']: + chat_parts.append(f"npc:{round_data['npc_message']}") + else: + # round 1~n 添加user_input和npc_message + if round_data['user_input']: + chat_parts.append(f"user:{round_data['user_input']}") + if round_data['npc_message']: + chat_parts.append(f"npc:{round_data['npc_message']}") + + chat_logs[session_id] = '\n'.join(chat_parts) + round_nums[session_id] = max_round + + except Exception as e: + logger.error(f"查询ES批次失败: {e}") + continue + + logger.info(f"完成ES查询，获取到 {len(chat_logs)} 个session的对话记录") + + # 添加chat_log和round_num字段 + self.data['chat_log'] = self.data['session_id'].map(chat_logs).fillna('') + self.data['round_num'] = self.data['session_id'].map(round_nums).fillna(0) + + logger.info("步骤三完成: 对话历史数据已补充") + + def export_final_excel(self): + """导出最终Excel文件""" + if self.data.empty: + logger.warning("数据为空，无法导出") + return + + logger.info("开始导出最终Excel文件...") + + # 确保字段顺序 + final_columns = [ + 'user_id', 'session_id', 'c_type', 'c_id', + 'play_result', 'updated_at', 'title', 'lesson', 'knowledge_points', + 'in_progress_goal', 'final_goal', 'reference_dialog', + 'chat_log', 'user_behavior_info', 'round_num' + ] + + # 重新排列列顺序 + self.data = self.data[final_columns] + + # 处理datetime字段，去掉时区信息（Excel不支持带时区的datetime） + if 'updated_at' in self.data.columns: + self.data['updated_at'] = pd.to_datetime(self.data['updated_at']).dt.tz_localize(None) + logger.info("最终导出时已处理updated_at字段的时区信息") + + # 生成文件名 + output_file = f"core_speaking_final_{START_DATE}_{END_DATE}.xlsx" + + # 导出Excel + self.data.to_excel(output_file, index=False) + + logger.info(f"最终Excel文件已导出: {output_file}") + logger.info(f"总计导出 {len(self.data)} 条记录") + + # 输出字段统计 + logger.info("字段完整性统计:") + for col in final_columns: + if col in ['chat_log', 'title', 'reference_dialog', 'user_behavior_info', + 'lesson', 'knowledge_points', 'in_progress_goal', 'final_goal']: + non_empty = (self.data[col] != '').sum() + logger.info(f" {col}: {non_empty}/{len(self.data)} 条记录有值") + elif col == 'round_num': + non_zero = (self.data[col] > 0).sum() + logger.info(f" {col}: {non_zero}/{len(self.data)} 条记录 > 0") + + def process(self): + """执行完整的数据处理流程""" + logger.info("="*60) + logger.info("开始口语核心互动数据处理") + logger.info(f"时间范围: {START_DATE} - {END_DATE}") + logger.info("="*60) + + try: + # 步骤一: PG数据筛选 + self.step1_extract_from_pg() + + # 步骤二: MySQL补充title + self.step2_add_title_from_mysql() + + # 步骤三: ES补充对话历史 + self.step3_add_chat_log_from_es() + + # 导出最终文件 + self.export_final_excel() + + logger.info("="*60) + logger.info("数据处理完成!") + logger.info("="*60) + + except Exception as e: + logger.error(f"数据处理过程中发生错误: {e}") + raise + +if __name__ == "__main__": + processor = CoreSpeakingDataProcessor() + processor.process() diff --git a/business_knowledge/git_scripts/extract_user_audio.py b/business_knowledge/git_scripts/extract_user_audio.py new file mode 100644 index 0000000..50c5080 --- /dev/null +++ b/business_knowledge/git_scripts/extract_user_audio.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +用户音频数据筛选脚本 +功能：从PostgreSQL数据库的分表(user_component_play_record_0~7)中提取指定时间段的用户音频数据。 +主要逻辑： +1. 数据源：遍历 user_component_play_record_0 至 user_component_play_record_7 表。 +2. 筛选条件： + - 时间范围：可配置 + - 数据有效性：user_behavior_info 非空且包含 userAudio 和 pronunciationScore。 +3. 采样规则： + - 目标总数：可配置 + - 用户限制：可配置 + - 随机策略：先随机打乱，再按用户分组限制，最后补齐或截断至目标数量。 +4. 输出：导出为Excel文件。 + 包含字段： + - index: 序号 + - source_table: 来源表名 + - created_at: 创建时间 + - user_id: 用户ID + - component_unique_code: 组件唯一标识 + - pronunciationScore: 发音评分 + - userAudio: 音频链接 + - expressContent: 朗读内容文本 +""" + +import os +import json +import re +import random +import psycopg2 +import pymysql +import pandas as pd +from datetime import datetime +from typing import List, Dict, Any +from dotenv import load_dotenv + +# 配置参数 +CONFIG = { + # 筛选时间范围 + 'START_TIME': '2025-11-10 00:00:00+08:00', + 'END_TIME': '2025-12-10 23:59:59+08:00', + + # 采样参数 + 'TARGET_TOTAL': 10000, # 目标总样本数 + 'MAX_PER_USER': 20, # 单个用户最大样本数 + 'TABLE_COUNT': 8, # 分表数量 (0~N-1) + + # 组件类型过滤 + 'C_TYPE_FILTER': 'mid_sentence_dialogue' # 仅筛选对话互动组件 +} + +class AudioDataExtractor: + def __init__(self): + # 加载环境变量 + load_dotenv() + + # PostgreSQL数据库连接配置 + self.db_config = { + 'host': os.getenv('PG_DB_HOST'), + 'port': os.getenv('PG_DB_PORT'), + 'user': os.getenv('PG_DB_USER'), + 'password': os.getenv('PG_DB_PASSWORD'), + 'database': os.getenv('PG_DB_DATABASE') + } + + # MySQL数据库连接配置 + self.mysql_config = { + 'host': os.getenv('MYSQL_HOST'), + 'user': os.getenv('MYSQL_USERNAME'), + 'password': os.getenv('MYSQL_PASSWORD'), + 'database': "vala_test", + 'port': int(os.getenv('MYSQL_PORT', 3306)), + 'charset': 'utf8mb4' + } + + # 分表名称列表 + self.table_names = [f'user_component_play_record_{i}' for i in range(CONFIG['TABLE_COUNT'])] + + + # 目标总数 + self.target_total = CONFIG['TARGET_TOTAL'] + # 每个用户最多记录数 + self.max_per_user = CONFIG['MAX_PER_USER'] + + def get_db_connection(self): + """获取数据库连接""" + try: + conn = psycopg2.connect(**self.db_config) + return conn + except Exception as e: + print(f"数据库连接失败: {e}") + raise + + def extract_audio_info(self, user_behavior_info: str) -> Dict[str, Any]: + """从user_behavior_info字段中提取音频信息""" + try: + behavior_data = json.loads(user_behavior_info) + if isinstance(behavior_data, list) and len(behavior_data) > 0: + # 取第一个元素 + data = behavior_data[0] + if 'userAudio' in data and 'pronunciationScore' in data: + return { + 'userAudio': data.get('userAudio'), + 'pronunciationScore': data.get('pronunciationScore'), + 'expressContent': data.get('expressContent') + } + except (json.JSONDecodeError, KeyError, IndexError): + pass + return {} + + def query_table_data(self, table_name: str) -> List[Dict]: + """查询单个表的数据""" + conn = self.get_db_connection() + cursor = conn.cursor() + + try: + query = f""" + SELECT user_id, component_unique_code, c_type, c_id, created_at, user_behavior_info + FROM {table_name} + WHERE created_at >= '{CONFIG['START_TIME']}' + AND created_at <= '{CONFIG['END_TIME']}' + AND c_type = '{CONFIG['C_TYPE_FILTER']}' + AND user_behavior_info IS NOT NULL + AND user_behavior_info != '' + """ + + cursor.execute(query) + rows = cursor.fetchall() + + results = [] + for row in rows: + user_id, component_unique_code, c_type, c_id, created_at, user_behavior_info = row + + # 提取音频信息 + audio_info = self.extract_audio_info(user_behavior_info) + if audio_info and 'userAudio' in audio_info and 'pronunciationScore' in audio_info: + results.append({ + 'source_table': table_name, + 'user_id': user_id, + 'component_unique_code': component_unique_code, + 'c_type': c_type, + 'c_id': c_id, + 'created_at': created_at, + 'userAudio': audio_info['userAudio'], + 'pronunciationScore': audio_info['pronunciationScore'], + 'expressContent': audio_info.get('expressContent') + }) + + return results + + finally: + cursor.close() + conn.close() + + def get_component_configs(self, data: List[Dict]) -> Dict[str, str]: + """从MySQL批量获取组件配置信息""" + # 提取所有unique的(c_type, c_id)组合 + unique_components = set() + for record in data: + if 'c_type' in record and 'c_id' in record: + unique_components.add((record['c_type'], record['c_id'])) + + if not unique_components: + print("没有需要查询的组件") + return {} + + print(f"正在从MySQL查询 {len(unique_components)} 个组件的配置信息...") + + # 连接MySQL + try: + conn = pymysql.connect(**self.mysql_config) + cursor = conn.cursor() + + # 存储组件配置的字典，key为"c_type-c_id" + component_configs = {} + + # 批量查询 + for c_type, c_id in unique_components: + query = """ + SELECT component_config + FROM middle_interaction_component + WHERE c_type = %s AND c_id = %s + """ + cursor.execute(query, (c_type, c_id)) + result = cursor.fetchone() + + if result and result[0]: + key = f"{c_type}-{c_id}" + component_configs[key] = result[0] + + cursor.close() + conn.close() + + print(f"成功查询到 {len(component_configs)} 个组件配置") + return component_configs + + except Exception as e: + print(f"查询MySQL组件配置失败: {e}") + return {} + + @staticmethod + def clean_text(text: str) -> str: + """清理文本：转小写，去除标点符号和空格""" + if not text: + return "" + # 转小写 + text = text.lower() + # 去除标点符号和特殊字符，只保留字母和数字 + text = re.sub(r'[^\w\s]', '', text) + # 去除多余空格 + text = re.sub(r'\s+', '', text) + return text + + @staticmethod + def levenshtein_distance(s1: str, s2: str) -> int: + """计算两个字符串的Levenshtein编辑距离""" + if len(s1) < len(s2): + return AudioDataExtractor.levenshtein_distance(s2, s1) + + if len(s2) == 0: + return len(s1) + + previous_row = range(len(s2) + 1) + for i, c1 in enumerate(s1): + current_row = [i + 1] + for j, c2 in enumerate(s2): + # 插入、删除、替换的成本 + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + previous_row = current_row + + return previous_row[-1] + + def parse_and_filter_by_config(self, data: List[Dict], component_configs: Dict[str, str]) -> List[Dict]: + """解析组件配置并筛选question.mode == 'read'的记录""" + print(f"\n开始根据组件配置筛选数据...") + print(f"筛选前数据量: {len(data)}") + + filtered_data = [] + skipped_no_config = 0 + skipped_invalid_json = 0 + skipped_wrong_mode = 0 + + for record in data: + c_type = record.get('c_type') + c_id = record.get('c_id') + + if not c_type or not c_id: + continue + + # 获取组件配置 + key = f"{c_type}-{c_id}" + config_str = component_configs.get(key) + + if not config_str: + skipped_no_config += 1 + continue + + try: + # 解析JSON配置 + config = json.loads(config_str) + + # 检查question.mode == "read" + question = config.get('question', {}) + mode = question.get('mode') + + if mode == 'read': + # 提取question.content作为refText + ref_text = question.get('content', '') + record['refText'] = ref_text + + # 计算编辑距离 + express_content = record.get('expressContent', '') + + # 清理文本（去除标点和大小写差异） + cleaned_express = self.clean_text(express_content) + cleaned_ref = self.clean_text(ref_text) + + # 计算编辑距离 + edit_distance = self.levenshtein_distance(cleaned_express, cleaned_ref) + record['editDistance'] = edit_distance + + # 计算相对编辑距离 + ref_len = len(cleaned_ref) + if ref_len > 0: + relative_edit_distance = round(edit_distance / ref_len, 4) + else: + relative_edit_distance = 0 + record['relativeEditDistance'] = relative_edit_distance + + filtered_data.append(record) + else: + skipped_wrong_mode += 1 + + except (json.JSONDecodeError, AttributeError, TypeError): + skipped_invalid_json += 1 + continue + + print(f"筛选后数据量: {len(filtered_data)}") + print(f" - 缺少配置: {skipped_no_config}") + print(f" - 配置解析失败: {skipped_invalid_json}") + print(f" - mode不是read: {skipped_wrong_mode}") + + return filtered_data + + def collect_all_data(self) -> List[Dict]: + """收集所有表的数据""" + all_data = [] + + for table_name in self.table_names: + print(f"正在查询表: {table_name}") + try: + table_data = self.query_table_data(table_name) + all_data.extend(table_data) + print(f"表 {table_name} 查询到 {len(table_data)} 条记录") + except Exception as e: + print(f"查询表 {table_name} 失败: {e}") + continue + + print(f"总共收集到 {len(all_data)} 条有效记录") + + if not all_data: + return [] + + # 从MySQL获取组件配置 + component_configs = self.get_component_configs(all_data) + + # 根据组件配置筛选数据（只保留question.mode == "read"的记录） + filtered_data = self.parse_and_filter_by_config(all_data, component_configs) + + return filtered_data + + def random_filter_data(self, data: List[Dict]) -> List[Dict]: + """随机筛选数据（不按评分分段控制）""" + # 随机打乱所有数据 + shuffled_data = data.copy() + random.shuffle(shuffled_data) + + print(f"开始随机筛选，总共 {len(shuffled_data)} 条记录") + return shuffled_data + + def apply_user_constraints(self, data: List[Dict]) -> List[Dict]: + """应用用户约束（每个用户最多2条）""" + user_records = {} + + # 按用户分组 + for record in data: + user_id = record['user_id'] + if user_id not in user_records: + user_records[user_id] = [] + user_records[user_id].append(record) + + # 每个用户最多选择2条 + final_data = [] + for user_id, records in user_records.items(): + if len(records) <= self.max_per_user: + final_data.extend(records) + else: + # 随机选择2条 + selected = random.sample(records, self.max_per_user) + final_data.extend(selected) + + return final_data + + def export_to_excel(self, data: List[Dict], filename: str = 'user_audio_data.xlsx'): + """导出数据到Excel文件""" + # 准备导出数据 + export_data = [] + for i, record in enumerate(data): + # 处理时区问题 - 转换为本地时间字符串 + created_at = record['created_at'] + if hasattr(created_at, 'tz_localize'): + created_at = created_at.tz_localize(None) + elif hasattr(created_at, 'replace'): + created_at = created_at.replace(tzinfo=None) + + export_data.append({ + 'index': i, + 'source_table': record['source_table'], + 'created_at': created_at, + 'user_id': record['user_id'], + 'component_unique_code': record['component_unique_code'], + 'c_type': record.get('c_type'), + 'c_id': record.get('c_id'), + 'pronunciationScore': record['pronunciationScore'], + 'userAudio': record['userAudio'], + 'expressContent': record.get('expressContent'), + 'refText': record.get('refText'), + 'editDistance': record.get('editDistance'), + 'relativeEditDistance': record.get('relativeEditDistance') + }) + + # 创建DataFrame并导出 + df = pd.DataFrame(export_data) + df.to_excel(filename, index=False) + print(f"数据已导出到: {filename}") + print(f"总共导出 {len(export_data)} 条记录") + + # 打印统计信息 + self.print_statistics(data) + + def print_statistics(self, data: List[Dict]): + """打印统计信息""" + print("\n=== 数据统计 ===") + + # 评分统计（显示分布情况但不按区间分组） + scores = [record['pronunciationScore'] for record in data] + print(f"\n评分统计:") + print(f" 总记录数: {len(scores)}") + print(f" 最高分: {max(scores)}") + print(f" 最低分: {min(scores)}") + print(f" 平均分: {sum(scores) / len(scores):.2f}") + + # 用户分布统计 + user_counts = {} + for record in data: + user_id = record['user_id'] + user_counts[user_id] = user_counts.get(user_id, 0) + 1 + + print(f"\n用户统计:") + print(f" 总用户数: {len(user_counts)}") + print(f" 平均每用户记录数: {len(data) / len(user_counts):.2f}") + + # 表分布统计 + table_counts = {} + for record in data: + table = record['source_table'] + table_counts[table] = table_counts.get(table, 0) + 1 + + print(f"\n表分布:") + for table, count in sorted(table_counts.items()): + print(f" {table}: {count} 条") + + def run(self): + """运行主流程""" + print("开始提取用户音频数据...") + + # 1. 收集所有数据 + all_data = self.collect_all_data() + + if not all_data: + print("未找到符合条件的数据") + return + + # 2. 随机筛选数据（不按评分分段控制） + filtered_data = self.random_filter_data(all_data) + + # 3. 应用用户约束 + final_data = self.apply_user_constraints(filtered_data) + + # 4. 如果数据不足500条，尝试补充 + if len(final_data) < self.target_total: + print(f"当前数据量 {len(final_data)} 条，少于目标 {self.target_total} 条") + # 从剩余数据中补充 + used_records = set((r['user_id'], r['component_unique_code'], str(r['created_at'])) for r in final_data) + available_data = [r for r in all_data if (r['user_id'], r['component_unique_code'], str(r['created_at'])) not in used_records] + + needed = self.target_total - len(final_data) + if len(available_data) >= needed: + additional = random.sample(available_data, needed) + final_data.extend(additional) + + # 5. 如果超过500条，随机选择500条 + if len(final_data) > self.target_total: + final_data = random.sample(final_data, self.target_total) + + # 6. 导出到Excel + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"user_audio_data_{timestamp}.xlsx" + self.export_to_excel(final_data, filename) + +def main(): + extractor = AudioDataExtractor() + extractor.run() + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/sample_unit_challenge_data_from_es.py b/business_knowledge/git_scripts/sample_unit_challenge_data_from_es.py new file mode 100644 index 0000000..16b33fc --- /dev/null +++ b/business_knowledge/git_scripts/sample_unit_challenge_data_from_es.py @@ -0,0 +1,463 @@ +""" +从es中 筛选用户数据 + +es相关配置通过以下环节变量 + +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + + +index: user-audio + +脚本思路: + +给定 一些过滤参数； 给定导出的excel文件名 （在脚本中以变量方式配置就行） + +导出我要的字段内容到一个 excel + +过滤字段: +timeStr: 字段内容为str 格式为: 2024-12-31 15:53:19 +期望支持配置 开始 日期 和 结束日期 （可以只配置一个 只配 开始日期 则筛选 >= 开始日期的记录， 只配结束日期 则筛选 <= 结束日期的记录） + +输出字段内容支持配置: + + +""" + +import os +from datetime import datetime +from dotenv import load_dotenv +from elasticsearch import Elasticsearch +import pandas as pd +import urllib.parse +from collections import defaultdict + +# 加载环境变量 +load_dotenv() + +# 配置参数 +INDEX_NAME = "llm_ai_tools_log" +OUTPUT_FILE = "单元挑战用户数据_250906_251024.xlsx" +START_DATE = "2025-09-06 00:00:00" # 开始日期，格式: YYYY-MM-DD HH:MM:SS，设为None则不限制 +END_DATE = "2025-10-24 00:00:00" # 结束日期，格式: YYYY-MM-DD HH:MM:SS，设为None则不限制 + +# type字段过滤配置：筛选指定类型的记录，为空则不限制 +FILTER_TYPES = ["sent_check_challenge", "speaking_topic_challenge"] + +# 可选的 userId 过滤配置：配置为[int, ...] 列表；为空则不限制 +FILTER_USER_IDS = [] # 例如: [123, 456] + +# 需要导出的字段 +EXPORT_FIELDS = [ + "type", + "question", + "user_answer", + "time_total_ms", + "score", + "is_passed", + "model", + "write_time_str", + "write_time_int", +] + + + +def create_es_client(): + """创建Elasticsearch客户端""" + # 获取环境变量并打印调试信息 + es_host = os.getenv('ES_HOST') + es_port = os.getenv('ES_PORT', 9200) + es_scheme = os.getenv('ES_SCHEME', 'https') + es_user = os.getenv('ES_USER') + es_password = os.getenv('ES_PASSWORD') + + print(f"[DEBUG] ES配置信息:") + print(f" ES_HOST: {es_host}") + print(f" ES_PORT: {es_port}") + print(f" ES_SCHEME: {es_scheme}") + print(f" ES_USER: {es_user}") + print(f" ES_PASSWORD: {'***已设置***' if es_password else '未设置'}") + + # 检查必要的环境变量 + if not es_host: + raise ValueError("ES_HOST环境变量未设置") + if not es_user: + raise ValueError("ES_USER环境变量未设置") + if not es_password: + raise ValueError("ES_PASSWORD环境变量未设置") + + # URL编码用户名和密码，处理特殊字符 + encoded_user = urllib.parse.quote(es_user, safe='') + encoded_password = urllib.parse.quote(es_password, safe='') + + print(f"[DEBUG] 原始密码包含特殊字符，已进行URL编码") + + # 方式1: 使用URL中嵌入认证信息 + host_url_with_auth = f"{es_scheme}://{encoded_user}:{encoded_password}@{es_host}:{es_port}" + print(f"[DEBUG] 连接URL (带认证): {es_scheme}://{encoded_user}:***@{es_host}:{es_port}") + + try: + # 尝试方式1: URL中嵌入认证 + es_config_1 = { + 'hosts': [host_url_with_auth], + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式1: URL中嵌入认证信息") + es_client = Elasticsearch(**es_config_1) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式1连接成功") + return es_client + + except Exception as e1: + print(f"[DEBUG] 方式1失败: {e1}") + + try: + # 尝试方式2: 使用basic_auth参数 + host_url = f"{es_scheme}://{es_host}:{es_port}" + es_config_2 = { + 'hosts': [host_url], + 'basic_auth': (es_user, es_password), + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式2: 使用basic_auth参数") + es_client = Elasticsearch(**es_config_2) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式2连接成功") + return es_client + + except Exception as e2: + print(f"[DEBUG] 方式2失败: {e2}") + + try: + # 尝试方式3: 使用http_auth参数 (旧版本兼容) + es_config_3 = { + 'hosts': [host_url], + 'http_auth': (es_user, es_password), + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式3: 使用http_auth参数") + es_client = Elasticsearch(**es_config_3) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式3连接成功") + return es_client + + except Exception as e3: + print(f"[DEBUG] 方式3失败: {e3}") + print(f"[ERROR] 所有认证方式都失败了") + raise e3 + +def build_query(start_date=None, end_date=None): + """构建ES查询条件""" + # 构建基础查询条件 + must_conditions = [] + + # 添加时间范围条件 + if start_date or end_date: + range_query = {} + + if start_date: + start_timestamp = int(datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").timestamp()) + range_query["gte"] = start_timestamp + print(f"[DEBUG] 开始时间戳: {start_timestamp} (对应 {start_date})") + + if end_date: + end_timestamp = int(datetime.strptime(end_date, "%Y-%m-%d %H:%M:%S").timestamp()) + range_query["lte"] = end_timestamp + print(f"[DEBUG] 结束时间戳: {end_timestamp} (对应 {end_date})") + + must_conditions.append({ + "range": { + "write_time_int": range_query + } + }) + + # 如果配置了 userId 列表，则仅选取对应 userId 的数据 + if FILTER_USER_IDS: + print(f"[DEBUG] 应用 userId 过滤: {FILTER_USER_IDS}") + must_conditions.append({ + "terms": { + "userId": FILTER_USER_IDS + } + }) + + # 如果配置了 type 列表，则仅选取对应 type 的数据 + if FILTER_TYPES: + print(f"[DEBUG] 应用 type 过滤: {FILTER_TYPES}") + must_conditions.append({ + "terms": { + "type": FILTER_TYPES + } + }) + + # 构建最终查询 + if must_conditions: + query = { + "bool": { + "must": must_conditions + } + } + else: + query = {"match_all": {}} + + print(f"[DEBUG] 查询条件: {query}") + + return { + "query": query, + "_source": EXPORT_FIELDS, + "sort": [{"write_time_int": {"order": "desc"}}] + } + +def fetch_data_from_es(es_client, start_date=None, end_date=None): + """从ES获取数据""" + query = build_query(start_date, end_date) + + try: + print(f"[DEBUG] 执行ES查询，使用scroll获取全量数据...") + + # 使用scroll API获取全量数据 + scroll_size = 1000 # 每次scroll获取的数据量 + scroll_timeout = '2m' # scroll超时时间 + + # 初始化scroll + query['size'] = scroll_size + response = es_client.search( + index=INDEX_NAME, + body=query, + scroll=scroll_timeout + ) + + scroll_id = response['_scroll_id'] + hits = response['hits']['hits'] + total_hits = response['hits']['total'] + + # 获取总数（兼容不同ES版本） + if isinstance(total_hits, dict): + total_count = total_hits['value'] + else: + total_count = total_hits + + print(f"[DEBUG] ES中匹配的总记录数: {total_count}") + + all_data = [] + batch_count = 1 + + # 处理第一批数据 + for hit in hits: + source = hit['_source'] + row = {} + for field in EXPORT_FIELDS: + row[field] = source.get(field, "") + all_data.append(row) + + print(f"[DEBUG] 已获取第 {batch_count} 批数据，当前总数: {len(all_data)}") + + # 继续scroll获取剩余数据 + while len(hits) == scroll_size: + batch_count += 1 + response = es_client.scroll(scroll_id=scroll_id, scroll=scroll_timeout) + scroll_id = response['_scroll_id'] + hits = response['hits']['hits'] + + for hit in hits: + source = hit['_source'] + row = {} + for field in EXPORT_FIELDS: + row[field] = source.get(field, "") + all_data.append(row) + + print(f"[DEBUG] 已获取第 {batch_count} 批数据，当前总数: {len(all_data)}") + + # 清理scroll + try: + es_client.clear_scroll(scroll_id=scroll_id) + except: + pass # 忽略清理错误 + + print(f"[DEBUG] 从ES获取到数据 {len(all_data)} 条记录") + return all_data + + except Exception as e: + print(f"查询ES时出错: {e}") + return [] + +def export_to_excel(data, filename): + """导出数据到Excel""" + if not data: + print("没有数据可导出") + return + + df = pd.DataFrame(data) + + try: + df.to_excel(filename, index=False, engine='openpyxl') + print(f"数据已导出到: {filename}") + print(f"共导出 {len(data)} 条记录") + except Exception as e: + print(f"导出Excel时出错: {e}") + +def debug_es_data(es_client): + """调试ES数据，了解实际数据情况""" + print("\n" + "="*60) + print("开始调试ES数据...") + + try: + # 1. 查询总数据量 + total_query = { + "query": {"match_all": {}}, + "size": 0 + } + response = es_client.search(index=INDEX_NAME, body=total_query) + total_count = response['hits']['total'] + if isinstance(total_count, dict): + total_count = total_count['value'] + print(f"[DEBUG] ES索引 '{INDEX_NAME}' 中总数据量: {total_count}") + + if total_count == 0: + print("[ERROR] ES索引中没有任何数据！") + return + + # 2. 查询最近的几条数据，了解数据结构 + sample_query = { + "query": {"match_all": {}}, + "size": 5, + "sort": [{"_id": {"order": "desc"}}] + } + response = es_client.search(index=INDEX_NAME, body=sample_query) + hits = response['hits']['hits'] + + print(f"[DEBUG] 获取到 {len(hits)} 条样本数据:") + for i, hit in enumerate(hits): + source = hit['_source'] + + print(f" 样本 {i+1}:") + print(f" write_time_int: {source.get('write_time_int', 'N/A')}") + print(f" timeStr: {source.get('timeStr', 'N/A')}") + print(f" type: {source.get('type', 'N/A')}") + print(f" userId: {source.get('userId', 'N/A')}") + + # 3. 查询时间范围内的数据 + time_range_query = { + "query": { + "range": { + "write_time_int": { + "gte": int(datetime.strptime(START_DATE, "%Y-%m-%d %H:%M:%S").timestamp()), + "lte": int(datetime.strptime(END_DATE, "%Y-%m-%d %H:%M:%S").timestamp()) + } + } + }, + "size": 0 + } + response = es_client.search(index=INDEX_NAME, body=time_range_query) + time_range_count = response['hits']['total'] + if isinstance(time_range_count, dict): + time_range_count = time_range_count['value'] + print(f"[DEBUG] 时间范围内数据量 ({START_DATE} 到 {END_DATE}): {time_range_count}") + + # 4. 查询时间范围的实际数据分布 + print(f"[DEBUG] 检查时间字段的实际值范围...") + agg_query = { + "query": {"match_all": {}}, + "size": 0, + "aggs": { + "time_stats": { + "stats": { + "field": "write_time_int" + } + } + } + } + response = es_client.search(index=INDEX_NAME, body=agg_query) + if 'aggregations' in response: + stats = response['aggregations']['time_stats'] + min_time = stats.get('min') + max_time = stats.get('max') + if min_time and max_time: + min_date = datetime.fromtimestamp(min_time).strftime("%Y-%m-%d %H:%M:%S") + max_date = datetime.fromtimestamp(max_time).strftime("%Y-%m-%d %H:%M:%S") + print(f" 最早时间: {min_date} (时间戳: {min_time})") + print(f" 最晚时间: {max_date} (时间戳: {max_time})") + + except Exception as e: + print(f"[ERROR] 调试ES数据时出错: {e}") + + print("="*60 + "\n") + +def main(): + """主函数""" + print("开始从ES获取单元挑战数据...") + print(f"索引: {INDEX_NAME}") + print(f"开始日期: {START_DATE if START_DATE else '不限制'}") + print(f"结束日期: {END_DATE if END_DATE else '不限制'}") + if FILTER_TYPES: + print(f"类型过滤: {FILTER_TYPES}") + if FILTER_USER_IDS: + print(f"用户ID过滤: {FILTER_USER_IDS}") + print("-" * 50) + + # 检查.env文件是否存在 + env_file = ".env" + if not os.path.exists(env_file): + print(f"[ERROR] {env_file} 文件不存在，请创建并配置ES连接信息") + print("参考 .env.example 文件进行配置") + return + + print(f"[DEBUG] 找到环境配置文件: {env_file}") + + # 创建ES客户端 + try: + es_client = create_es_client() + except ValueError as e: + print(f"[ERROR] 配置错误: {e}") + print("请检查 .env 文件中的ES配置") + return + except Exception as e: + print(f"[ERROR] 创建ES客户端失败: {e}") + return + + # 测试连接 + try: + print("[DEBUG] 正在测试ES连接...") + # ES客户端创建函数中已经包含了连接测试，这里不需要重复测试 + print(f"[SUCCESS] ES连接已建立") + except Exception as e: + print(f"[ERROR] ES连接失败: {e}") + print("\n可能的解决方案:") + print("1. 检查ES服务是否正常运行") + print("2. 验证.env文件中的ES_HOST、ES_USER、ES_PASSWORD是否正确") + print("3. 确认网络连接是否正常") + print("4. 检查ES用户权限是否足够") + print("5. 密码中包含特殊字符，已尝试URL编码处理") + return + + # 获取数据 + data = fetch_data_from_es(es_client, START_DATE, END_DATE) + + # 导出到Excel + if data: + export_to_excel(data, OUTPUT_FILE) + else: + print("未获取到任何数据") + +if __name__ == "__main__": + main() diff --git a/business_knowledge/git_scripts/sample_user_data_from_es.py b/business_knowledge/git_scripts/sample_user_data_from_es.py new file mode 100644 index 0000000..3a1e415 --- /dev/null +++ b/business_knowledge/git_scripts/sample_user_data_from_es.py @@ -0,0 +1,599 @@ +""" +从es中采样用户数据 + +es相关配置通过以下环节变量 + +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + + +index: user-audio + +脚本思路: + +给定 一些过滤参数； 给定导出的excel文件名 （在脚本中以变量方式配置就行） + +导出我要的字段内容到一个 excel + +过滤字段: +timeStr: 字段内容为str 格式为: 2024-12-31 15:53:19 +期望支持配置 开始 日期 和 结束日期 （可以只配置一个 只配 开始日期 则筛选 >= 开始日期的记录， 只配结束日期 则筛选 <= 结束日期的记录） + +输出以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +""" + +import os +from datetime import datetime +from dotenv import load_dotenv +from elasticsearch import Elasticsearch +import pandas as pd +import urllib.parse +import re +from collections import defaultdict + +# 加载环境变量 +load_dotenv() + +# 配置参数 +INDEX_NAME = os.getenv("ES_INDEX", "user-audio") +OUTPUT_FILE = "user_audio_data.xlsx" +START_DATE = "2025-10-15 00:00:00" # 开始日期，格式: YYYY-MM-DD HH:MM:SS，设为None则不限制 +END_DATE = "2025-10-17 00:00:00" # 结束日期，格式: YYYY-MM-DD HH:MM:SS，设为None则不限制 + +# 可选的 userId 过滤配置：配置为[int, ...] 列表；为空则不限制 +FILTER_USER_IDS = [356] # 例如: [123, 456] + +# 采样配置参数 +MAX_SAMPLES_PER_USER_MSG = 50 # 每个不重复的userMsg最多采样的数据条数 +MAX_SAMPLES_PER_USER_ID = 20 # 每个userId最多采样的数据条数 + +# 需要导出的字段 +EXPORT_FIELDS = [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr" +] + +def create_es_client(): + """创建Elasticsearch客户端""" + # 获取环境变量并打印调试信息 + es_host = os.getenv('ES_HOST') + es_port = os.getenv('ES_PORT', 9200) + es_scheme = os.getenv('ES_SCHEME', 'https') + es_user = os.getenv('ES_USER') + es_password = os.getenv('ES_PASSWORD') + + print(f"[DEBUG] ES配置信息:") + print(f" ES_HOST: {es_host}") + print(f" ES_PORT: {es_port}") + print(f" ES_SCHEME: {es_scheme}") + print(f" ES_USER: {es_user}") + print(f" ES_PASSWORD: {'***已设置***' if es_password else '未设置'}") + + # 检查必要的环境变量 + if not es_host: + raise ValueError("ES_HOST环境变量未设置") + if not es_user: + raise ValueError("ES_USER环境变量未设置") + if not es_password: + raise ValueError("ES_PASSWORD环境变量未设置") + + # URL编码用户名和密码，处理特殊字符 + encoded_user = urllib.parse.quote(es_user, safe='') + encoded_password = urllib.parse.quote(es_password, safe='') + + print(f"[DEBUG] 原始密码包含特殊字符，已进行URL编码") + + # 方式1: 使用URL中嵌入认证信息 + host_url_with_auth = f"{es_scheme}://{encoded_user}:{encoded_password}@{es_host}:{es_port}" + print(f"[DEBUG] 连接URL (带认证): {es_scheme}://{encoded_user}:***@{es_host}:{es_port}") + + try: + # 尝试方式1: URL中嵌入认证 + es_config_1 = { + 'hosts': [host_url_with_auth], + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式1: URL中嵌入认证信息") + es_client = Elasticsearch(**es_config_1) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式1连接成功") + return es_client + + except Exception as e1: + print(f"[DEBUG] 方式1失败: {e1}") + + try: + # 尝试方式2: 使用basic_auth参数 + host_url = f"{es_scheme}://{es_host}:{es_port}" + es_config_2 = { + 'hosts': [host_url], + 'basic_auth': (es_user, es_password), + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式2: 使用basic_auth参数") + es_client = Elasticsearch(**es_config_2) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式2连接成功") + return es_client + + except Exception as e2: + print(f"[DEBUG] 方式2失败: {e2}") + + try: + # 尝试方式3: 使用http_auth参数 (旧版本兼容) + es_config_3 = { + 'hosts': [host_url], + 'http_auth': (es_user, es_password), + 'verify_certs': False, + 'ssl_show_warn': False, + 'request_timeout': 30, + 'retry_on_timeout': True + } + + print("[DEBUG] 尝试方式3: 使用http_auth参数") + es_client = Elasticsearch(**es_config_3) + + # 测试连接 + info = es_client.info() + print(f"[SUCCESS] 方式3连接成功") + return es_client + + except Exception as e3: + print(f"[DEBUG] 方式3失败: {e3}") + print(f"[ERROR] 所有认证方式都失败了") + raise e3 + +def build_query(start_date=None, end_date=None): + """构建ES查询条件""" + # 构建基础查询条件 + must_conditions = [] + + # 添加时间范围条件 + if start_date or end_date: + range_query = {} + + if start_date: + start_timestamp = int(datetime.strptime(start_date, "%Y-%m-%d %H:%M:%S").timestamp()) + range_query["gte"] = start_timestamp + print(f"[DEBUG] 开始时间戳: {start_timestamp} (对应 {start_date})") + + if end_date: + end_timestamp = int(datetime.strptime(end_date, "%Y-%m-%d %H:%M:%S").timestamp()) + range_query["lte"] = end_timestamp + print(f"[DEBUG] 结束时间戳: {end_timestamp} (对应 {end_date})") + + must_conditions.append({ + "range": { + "timeInt": range_query + } + }) + + # 如果配置了 userId 列表，则仅选取对应 userId 的数据 + if FILTER_USER_IDS: + print(f"[DEBUG] 应用 userId 过滤: {FILTER_USER_IDS}") + must_conditions.append({ + "terms": { + "userId": FILTER_USER_IDS + } + }) + + # 移除soeData的exists查询，改为在应用层进行更精确的过滤 + # 注释掉原来的soeData exists查询 + # must_conditions.append({ + # "exists": { + # "field": "soeData" + # } + # }) + + # 构建最终查询 + if must_conditions: + query = { + "bool": { + "must": must_conditions + } + } + else: + query = {"match_all": {}} + + print(f"[DEBUG] 查询条件: {query}") + + return { + "query": query, + "_source": EXPORT_FIELDS, + "sort": [{"timeInt": {"order": "desc"}}] + } + +def fetch_data_from_es(es_client, start_date=None, end_date=None): + """从ES获取数据""" + query = build_query(start_date, end_date) + + try: + print(f"[DEBUG] 执行ES查询，使用scroll获取全量数据...") + + # 使用scroll API获取全量数据 + scroll_size = 1000 # 每次scroll获取的数据量 + scroll_timeout = '2m' # scroll超时时间 + + # 初始化scroll + query['size'] = scroll_size + response = es_client.search( + index=INDEX_NAME, + body=query, + scroll=scroll_timeout + ) + + scroll_id = response['_scroll_id'] + hits = response['hits']['hits'] + total_hits = response['hits']['total'] + + # 获取总数（兼容不同ES版本） + if isinstance(total_hits, dict): + total_count = total_hits['value'] + else: + total_count = total_hits + + print(f"[DEBUG] ES中匹配的总记录数: {total_count}") + + all_data = [] + batch_count = 1 + + # 处理第一批数据 + for hit in hits: + source = hit['_source'] + row = {} + for field in EXPORT_FIELDS: + row[field] = source.get(field, "") + all_data.append(row) + + print(f"[DEBUG] 已获取第 {batch_count} 批数据，当前总数: {len(all_data)}") + + # 继续scroll获取剩余数据 + while len(hits) == scroll_size: + batch_count += 1 + response = es_client.scroll(scroll_id=scroll_id, scroll=scroll_timeout) + scroll_id = response['_scroll_id'] + hits = response['hits']['hits'] + + for hit in hits: + source = hit['_source'] + row = {} + for field in EXPORT_FIELDS: + row[field] = source.get(field, "") + all_data.append(row) + + print(f"[DEBUG] 已获取第 {batch_count} 批数据，当前总数: {len(all_data)}") + + # 清理scroll + try: + es_client.clear_scroll(scroll_id=scroll_id) + except: + pass # 忽略清理错误 + + print(f"[DEBUG] 从ES获取到原始数据 {len(all_data)} 条记录") + + # 根据是否配置了 userId 列表决定是否跳过过滤与采样逻辑 + if FILTER_USER_IDS: + print("[DEBUG] 已配置 userId 列表，跳过过滤与采样逻辑，返回全部匹配数据") + return all_data + else: + # 应用过滤和采样逻辑 + filtered_sampled_data = filter_and_sample_data(all_data) + return filtered_sampled_data + + except Exception as e: + print(f"查询ES时出错: {e}") + return [] + +def export_to_excel(data, filename): + """导出数据到Excel""" + if not data: + print("没有数据可导出") + return + + df = pd.DataFrame(data) + + # 生成带时间戳的文件名 + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + base_name = filename.rsplit('.', 1)[0] + extension = filename.rsplit('.', 1)[1] if '.' in filename else 'xlsx' + timestamped_filename = f"{base_name}_{timestamp}.{extension}" + + try: + df.to_excel(timestamped_filename, index=False, engine='openpyxl') + print(f"数据已导出到: {timestamped_filename}") + print(f"共导出 {len(data)} 条记录") + except Exception as e: + print(f"导出Excel时出错: {e}") + +def contains_chinese(text): + """检测文本是否包含中文字符""" + if not text: + return False + chinese_pattern = re.compile(r'[\u4e00-\u9fff]') + return bool(chinese_pattern.search(text)) + +def filter_and_sample_data(data): + """过滤和采样数据""" + print(f"[DEBUG] 开始过滤和采样，原始数据量: {len(data)}") + + # 第一步：过滤数据 + filtered_data = [] + soe_data_empty_count = 0 + soe_data_not_json_count = 0 + chinese_msg_count = 0 + + for i, item in enumerate(data): + # 检查soeData是否存在且以"{"开头 + soe_data = item.get('soeData', '') + if not soe_data: + soe_data_empty_count += 1 + if i < 5: # 只打印前5个样本的详细信息 + print(f"[DEBUG] 样本 {i+1}: soeData为空或不存在") + continue + + if not str(soe_data).strip().startswith('{'): + soe_data_not_json_count += 1 + if i < 5: # 只打印前5个样本的详细信息 + print(f"[DEBUG] 样本 {i+1}: soeData不以'{{' 开头，内容: {str(soe_data)[:100]}...") + continue + + # 检查userMsg是否不包含中文 + user_msg = item.get('userMsg', '') + if contains_chinese(user_msg): + chinese_msg_count += 1 + if i < 5: # 只打印前5个样本的详细信息 + print(f"[DEBUG] 样本 {i+1}: userMsg包含中文，内容: {user_msg[:50]}...") + continue + + filtered_data.append(item) + if i < 5: # 只打印前5个样本的详细信息 + print(f"[DEBUG] 样本 {i+1}: 通过过滤，userMsg: {user_msg[:50]}...") + + print(f"[DEBUG] 过滤统计:") + print(f" - soeData为空: {soe_data_empty_count} 条") + print(f" - soeData不以'{{' 开头: {soe_data_not_json_count} 条") + print(f" - userMsg包含中文: {chinese_msg_count} 条") + print(f" - 通过过滤的数据: {len(filtered_data)} 条") + + # 第二步：按userMsg分组采样 + user_msg_groups = defaultdict(list) + for item in filtered_data: + user_msg = item.get('userMsg', '') + user_msg_groups[user_msg].append(item) + + print(f"[DEBUG] 不重复的userMsg数量: {len(user_msg_groups)}") + + # 对每个userMsg组进行采样 + sampled_by_msg = [] + for user_msg, items in user_msg_groups.items(): + # 每个userMsg最多取MAX_SAMPLES_PER_USER_MSG条 + sampled_items = items[:MAX_SAMPLES_PER_USER_MSG] + sampled_by_msg.extend(sampled_items) + if len(items) > MAX_SAMPLES_PER_USER_MSG: + print(f"[DEBUG] userMsg '{user_msg}' 有 {len(items)} 条数据，采样了 {MAX_SAMPLES_PER_USER_MSG} 条") + + print(f"[DEBUG] 按userMsg采样后数据量: {len(sampled_by_msg)}") + + # 第三步：按userId分组采样 + user_id_groups = defaultdict(list) + for item in sampled_by_msg: + user_id = item.get('userId', '') + user_id_groups[user_id].append(item) + + print(f"[DEBUG] 不重复的userId数量: {len(user_id_groups)}") + + # 对每个userId组进行采样 + final_sampled_data = [] + for user_id, items in user_id_groups.items(): + # 每个userId最多取MAX_SAMPLES_PER_USER_ID条 + sampled_items = items[:MAX_SAMPLES_PER_USER_ID] + final_sampled_data.extend(sampled_items) + if len(items) > MAX_SAMPLES_PER_USER_ID: + print(f"[DEBUG] userId '{user_id}' 有 {len(items)} 条数据，采样了 {MAX_SAMPLES_PER_USER_ID} 条") + + print(f"[DEBUG] 最终采样数据量: {len(final_sampled_data)}") + + return final_sampled_data + +def debug_es_data(es_client): + """调试ES数据，了解实际数据情况""" + print("\n" + "="*60) + print("开始调试ES数据...") + + try: + # 1. 查询总数据量 + total_query = { + "query": {"match_all": {}}, + "size": 0 + } + response = es_client.search(index=INDEX_NAME, body=total_query) + total_count = response['hits']['total'] + if isinstance(total_count, dict): + total_count = total_count['value'] + print(f"[DEBUG] ES索引 '{INDEX_NAME}' 中总数据量: {total_count}") + + if total_count == 0: + print("[ERROR] ES索引中没有任何数据！") + return + + # 2. 查询最近的几条数据，了解数据结构 + sample_query = { + "query": {"match_all": {}}, + "size": 5, + "sort": [{"_id": {"order": "desc"}}] + } + response = es_client.search(index=INDEX_NAME, body=sample_query) + hits = response['hits']['hits'] + + print(f"[DEBUG] 获取到 {len(hits)} 条样本数据:") + for i, hit in enumerate(hits): + source = hit['_source'] + soe_data = source.get('soeData', '') + soe_data_preview = str(soe_data)[:100] if soe_data else 'N/A' + soe_data_starts_with_brace = str(soe_data).strip().startswith('{') if soe_data else False + + print(f" 样本 {i+1}:") + print(f" timeInt: {source.get('timeInt', 'N/A')}") + print(f" timeStr: {source.get('timeStr', 'N/A')}") + print(f" soeData存在: {'是' if soe_data else '否'}") + print(f" soeData以{{开头: {'是' if soe_data_starts_with_brace else '否'}") + print(f" soeData预览: {soe_data_preview}...") + print(f" userMsg: {source.get('userMsg', 'N/A')[:50]}...") + print(f" userId: {source.get('userId', 'N/A')}") + + # 3. 查询时间范围内的数据（不加soeData过滤） + time_range_query = { + "query": { + "range": { + "timeInt": { + "gte": int(datetime.strptime(START_DATE, "%Y-%m-%d %H:%M:%S").timestamp()), + "lte": int(datetime.strptime(END_DATE, "%Y-%m-%d %H:%M:%S").timestamp()) + } + } + }, + "size": 0 + } + response = es_client.search(index=INDEX_NAME, body=time_range_query) + time_range_count = response['hits']['total'] + if isinstance(time_range_count, dict): + time_range_count = time_range_count['value'] + print(f"[DEBUG] 时间范围内数据量 ({START_DATE} 到 {END_DATE}): {time_range_count}") + + # 4. 查询有soeData的数据总量 + soe_data_query = { + "query": { + "exists": { + "field": "soeData" + } + }, + "size": 0 + } + response = es_client.search(index=INDEX_NAME, body=soe_data_query) + soe_data_count = response['hits']['total'] + if isinstance(soe_data_count, dict): + soe_data_count = soe_data_count['value'] + print(f"[DEBUG] 有soeData字段的数据总量: {soe_data_count}") + + # 5. 查询时间范围的实际数据分布 + print(f"[DEBUG] 检查时间字段的实际值范围...") + agg_query = { + "query": {"match_all": {}}, + "size": 0, + "aggs": { + "time_stats": { + "stats": { + "field": "timeInt" + } + } + } + } + response = es_client.search(index=INDEX_NAME, body=agg_query) + if 'aggregations' in response: + stats = response['aggregations']['time_stats'] + min_time = stats.get('min') + max_time = stats.get('max') + if min_time and max_time: + min_date = datetime.fromtimestamp(min_time).strftime("%Y-%m-%d %H:%M:%S") + max_date = datetime.fromtimestamp(max_time).strftime("%Y-%m-%d %H:%M:%S") + print(f" 最早时间: {min_date} (时间戳: {min_time})") + print(f" 最晚时间: {max_date} (时间戳: {max_time})") + + except Exception as e: + print(f"[ERROR] 调试ES数据时出错: {e}") + + print("="*60 + "\n") + +def main(): + """主函数""" + print("开始从ES采样用户数据...") + print(f"索引: {INDEX_NAME}") + print(f"开始日期: {START_DATE if START_DATE else '不限制'}") + print(f"结束日期: {END_DATE if END_DATE else '不限制'}") + if FILTER_USER_IDS: + print(f"userId过滤: {FILTER_USER_IDS}") + print("在配置了 userId 的情况下，将导出匹配用户的全部数据，跳过其他过滤与采样") + else: + print(f"过滤条件: soeData非空 且 userMsg不包含中文") + print(f"采样配置: 每个userMsg最多{MAX_SAMPLES_PER_USER_MSG}条，每个userId最多{MAX_SAMPLES_PER_USER_ID}条") + print("-" * 50) + + # 检查.env文件是否存在 + env_file = ".env" + if not os.path.exists(env_file): + print(f"[ERROR] {env_file} 文件不存在，请创建并配置ES连接信息") + print("参考 .env.example 文件进行配置") + return + + print(f"[DEBUG] 找到环境配置文件: {env_file}") + + # 创建ES客户端 + try: + es_client = create_es_client() + except ValueError as e: + print(f"[ERROR] 配置错误: {e}") + print("请检查 .env 文件中的ES配置") + return + except Exception as e: + print(f"[ERROR] 创建ES客户端失败: {e}") + return + + # 测试连接 + try: + print("[DEBUG] 正在测试ES连接...") + # ES客户端创建函数中已经包含了连接测试，这里不需要重复测试 + print(f"[SUCCESS] ES连接已建立") + except Exception as e: + print(f"[ERROR] ES连接失败: {e}") + print("\n可能的解决方案:") + print("1. 检查ES服务是否正常运行") + print("2. 验证.env文件中的ES_HOST、ES_USER、ES_PASSWORD是否正确") + print("3. 确认网络连接是否正常") + print("4. 检查ES用户权限是否足够") + print("5. 密码中包含特殊字符，已尝试URL编码处理") + return + + # 获取数据 + data = fetch_data_from_es(es_client, START_DATE, END_DATE) + + # 导出到Excel + if data: + export_to_excel(data, OUTPUT_FILE) + else: + print("未获取到任何数据") + +if __name__ == "__main__": + main() diff --git a/business_knowledge/knowledge_summary.md b/business_knowledge/knowledge_summary.md new file mode 100644 index 0000000..78e012a --- /dev/null +++ b/business_knowledge/knowledge_summary.md @@ -0,0 +1,149 @@ +# 业务知识库总结 + +## 整体业务理解 + +### 公司业务模式 +这是一个在线教育产品，主要提供 L1/L2 级别的英语学习课程。 + +### 核心业务流程 +1. **用户获取**：用户通过各个渠道下载 App 并注册 +2. **用户激活**：用户创建角色，填写性别、生日等信息 +3. **用户转化**：用户通过站内或站外渠道购课 +4. **用户学习**：用户学习课程，完成课时 +5. **数据回收**：收集用户学习行为数据，用于分析和优化 + +--- + +## 核心数据模型 + +### 1. 用户层 +**表**：`bi_vala_app_account` +- 记录用户注册信息 +- 关键字段：id, created_at, download_channel, key_from, status +- 筛选条件：status=1, deleted_at IS NULL, 排除测试用户ID + +### 2. 用户详情层 +**表**：`account_detail_info` +- 记录用户的详细信息 +- 关键字段：account_id, login_address, phone_login_times +- login_address 格式："省份-城市" + +### 3. 角色层 +**表**：`bi_vala_app_character` +- 一个用户可以有多个角色 +- 关键字段：id, account_id, gender, birthday, purchase_season_package, created_at +- 性别映射：0=girl, 1=boy, 其他=unknow +- 赛季包状态：'[1]'=未购买，其他=已购买 + +### 4. 订单层 +**表**：`bi_vala_order` +- 记录用户购课订单 +- 关键字段：account_id, sale_channel, key_from, pay_success_date, pay_amount, pay_amount_int, order_status, goods_name +- 有效订单筛选：order_status=3 AND pay_amount_int>49800 +- 购课渠道：17个渠道映射 + +### 5. 课程层 +**表**：`bi_level_unit_lesson` +- 课程体系映射表 +- 课程层级结构：course_level (L1/L2) → course_season (S0-S4) → course_unit (U00-U48) → course_lesson (L1-L5) +- chapter_id 映射到完整的课程ID + +### 6. 学习行为层 +**表**：`bi_user_chapter_play_record_0~7`（8个分表） +- 记录用户的课程播放记录 +- 关键字段：user_id, chapter_id, chapter_unique_id, play_status, updated_at, created_at +- play_status=1 表示播放完成 +- 需要用 UNION ALL 合并8个分表 + +**表**：`bi_user_component_play_record_0~7`（8个分表） +- 记录用户的组件播放记录（更细粒度） +- 关键字段：chapter_unique_id, interval_time（毫秒） +- 用于计算完课耗时 + +--- + +## 核心业务指标 + +### 1. 用户指标 +- **新增注册用户数**：按日期、渠道统计 +- **用户画像**：性别、年龄、地域分布 + +### 2. 转化指标 +- **转化率**：注册 → 购课的转化 +- **购课标签**：未购课、站外购课、站内购课 +- **退费率**：订单退费情况 + +### 3. 收入指标 +- **GMV**：成交总额，按渠道、日期统计 +- **购课金额**：客单价分析 + +### 4. 学习行为指标 +- **课程进入完成率**：进入课程 → 完成课程的转化 +- **平均通关时长**：课程完课平均时间 +- **学习进度**：用户完课的课程数量和顺序 +- **完课间隔**：距离上次完课的时间 + +--- + +## 常用分析模式 + +### 1. 用户全链路分析 +将用户、角色、订单、课程完课数据关联，形成宽表，用于综合分析。 + +### 2. 渠道分析 +按 download_channel 或 sale_channel 分组，分析不同渠道的用户质量和转化效果。 + +### 3. 课程分析 +分析不同课程的完课率、完课时长，识别热门课程和难点课程。 + +### 4. 时间序列分析 +按日期分组，分析用户增长、收入、学习行为的趋势变化。 + +--- + +## 常见筛选条件 + +### 测试用户排除 +```sql +id not in (51, 2121, 1386, 1397, ...) +``` + +### 有效订单 +```sql +order_status = 3 +AND pay_amount_int > 49800 +``` + +### 有效用户 +```sql +status = 1 +AND deleted_at IS NULL +``` + +### 完课记录 +```sql +play_status = 1 +``` + +--- + +## 数据处理技巧 + +### 1. 分表合并 +使用 UNION ALL 合并8个分表： +```sql +select * from bi_user_chapter_play_record_0 +union all +select * from bi_user_chapter_play_record_1 +-- ... 其他6个表 +``` + +### 2. 渠道映射 +使用 CASE WHEN 将数字编码映射为渠道名称。 + +### 3. 时间处理 +- 使用 `date()` 或 `to_char()` 提取日期 +- 使用 `interval_time/1000/60` 将毫秒转为分钟 + +### 4. 去重逻辑 +使用 `rank() over (partition by ... order by ...)` 取第一条记录。 diff --git a/business_knowledge/sql_queries/README.md b/business_knowledge/sql_queries/README.md new file mode 100644 index 0000000..7f7029e --- /dev/null +++ b/business_knowledge/sql_queries/README.md @@ -0,0 +1,19 @@ +# SQL 查询文档索引 + +创建时间: 2026-03-02 18:04:16 + +## 文档列表 + +- [全字段大表](全字段大表.md) +- [平均通关时长](平均通关时长.md) +- [新增注册用户数by渠道](新增注册用户数by渠道.md) +- [课程进入完成率](课程进入完成率.md) +- [账号角色年龄地址](账号角色年龄地址.md) +- [退费率](退费率.md) +- [销转学习进度](销转学习进度.md) +- [班主任关注数据](班主任关注数据.md) +- [端内GMV](端内GMV.md) +- [端内用户课程进入完成率](端内用户课程进入完成率.md) +- [端内购课用户学习行为](端内购课用户学习行为.md) +- [转化率](转化率.md) +- [课程ID映射](课程ID映射.md) diff --git a/business_knowledge/sql_queries/全字段大表.md b/business_knowledge/sql_queries/全字段大表.md new file mode 100644 index 0000000..4403e73 --- /dev/null +++ b/business_knowledge/sql_queries/全字段大表.md @@ -0,0 +1,292 @@ +# 全字段大表 + +**获取时间:** 2026-03-02 +**飞书文档 Token:** VVyWd5491o6tuqxceCVci6dVnFd + +## 业务说明 + +这个查询将用户、购课、角色、课程完课等多个维度的数据整合在一起，形成一个宽表，适合进行综合分析。 + +## 涉及的数据表 + +1. **bi_vala_app_account** - 用户账号表 +2. **account_detail_info** - 账号详情表 +3. **bi_vala_order** - 订单表 +4. **bi_vala_app_character** - 角色表 +5. **bi_user_chapter_play_record_0~7** - 用户章节播放记录表（分表） +6. **bi_level_unit_lesson** - 课程单元表 +7. **bi_user_component_play_record_0~7** - 用户组件播放记录表（分表） + +## SQL 查询 + +```sql +select a.id as "用户ID" + ,a.created_date as "注册日期" + ,a.download_channel as "下载渠道" + ,a.key_from as "下载key_from" + ,b.login_address as "城市" + ,b.phone_login as "是否手机登录" + ,c.sale_channel as "购课渠道" + ,case when c.sale_channel is NULL then '未购课' + when c.sale_channel = '站外' then '站外购课' + else '站内购课' + end as "购课标签" + ,c.key_from as "购课key_from" + ,c.pay_date as "购课日期" + ,c.pay_amount as "购课金额" + ,d.id as "角色ID" + ,d.characer_pay_status as "角色是否付费" + ,d.gender as "性别" + ,2026 - cast(d.birthday as int) as "年龄" + ,e.chapter_id as "课程ID" + ,e.course_id as "课程名称" + ,e.chapter_unique_id as "完课标识" + ,e.finish_date as "完课日期" + ,e.finish_time as "完课耗时" +from +( + select id + ,key_from + ,to_char(created_at,'YYYY-MM-DD') as created_date + ,download_channel + from bi_vala_app_account + where status = 1 + and id not in (51,2121) + and deleted_at is NULL + group by id + ,key_from + ,created_at + ,download_channel +) as a +left join +( + select account_id + ,split_part(login_address,'-',2) as login_address + ,case when phone_login_times = 0 then 0 + else 1 + end as phone_login + from account_detail_info + group by account_id + ,login_address + ,case when phone_login_times = 0 then 0 + else 1 + end +) as b on a.id = b.account_id +left join +( + select account_id + ,case when sale_channel = 11 then '苹果' + when sale_channel = 12 then '华为' + when sale_channel = 13 then '小米' + when sale_channel = 14 then '荣耀' + when sale_channel = 15 then '应用宝' + when sale_channel = 17 then '魅族' + when sale_channel = 18 then 'VIVO' + when sale_channel = 19 then 'OPPO' + when sale_channel = 21 then '学而思' + when sale_channel = 22 then '讯飞' + when sale_channel = 23 then '步步高' + when sale_channel = 24 then '作业帮' + when sale_channel = 25 then '小度' + when sale_channel = 26 then '希沃' + when sale_channel = 27 then '京东方' + when sale_channel = 41 then '官网' + when sale_channel = 71 then '小程序' + else '站外' + end as sale_channel + ,key_from + ,to_char(pay_success_date,'YYYY-MM-DD') as pay_date + ,pay_amount + from bi_vala_order + where order_status = 3 + and pay_amount_int > 49800 + group by account_id + ,case when sale_channel = 11 then '苹果' + when sale_channel = 12 then '华为' + when sale_channel = 13 then '小米' + when sale_channel = 14 then '荣耀' + when sale_channel = 15 then '应用宝' + when sale_channel = 17 then '魅族' + when sale_channel = 18 then 'VIVO' + when sale_channel = 19 then 'OPPO' + when sale_channel = 21 then '学而思' + when sale_channel = 22 then '讯飞' + when sale_channel = 23 then '步步高' + when sale_channel = 24 then '作业帮' + when sale_channel = 25 then '小度' + when sale_channel = 26 then '希沃' + when sale_channel = 27 then '京东方' + when sale_channel = 41 then '官网' + when sale_channel = 71 then '小程序' + else '站外' + end + ,key_from + ,pay_success_date + ,pay_amount +) as c on a.id = c.account_id +left join +( + select id + ,account_id + ,case when purchase_season_package = '[1]' then 0 + else 1 + end as characer_pay_status + ,case when gender = 0 then 'girl' + when gender = 1 then 'boy' + else 'unknow' + end as gender + ,case when split_part(birthday,'-',1) = '' then '0000' + else split_part(birthday,'-',1) + end as birthday + from bi_vala_app_character + where deleted_at is NULL + group by id + ,account_id + ,case when purchase_season_package = '[1]' then 0 + else 1 + end + ,case when gender = 0 then 'girl' + when gender = 1 then 'boy' + else 'unknow' + end + ,case when split_part(birthday,'-',1) = '' then '0000' + else split_part(birthday,'-',1) + end +) as d on a.id = d.account_id +left join +( + select user_id + ,chapter_id + ,format('%s-%s-%s-%s',course_level,course_season,course_unit,course_lesson) as course_id + ,x.chapter_unique_id + ,finish_date + ,format('%s:%s',floor(sum(interval_time)/1000/60),mod((sum(interval_time)/1000),60)) as finish_time + ,rank () over (partition by x.chapter_unique_id order by finish_date) as rankno + from + ( + select user_id + ,chapter_id + ,chapter_unique_id + ,to_char(updated_at,'YYYY-MM-DD') as finish_date + from bi_user_chapter_play_record_0 + where chapter_id in (55,56,57,58,59) + and play_status = 1 + group by id + ,user_id + ,chapter_id + ,chapter_unique_id + ,updated_at + union all + select user_id + ,chapter_id + ,chapter_unique_id + ,to_char(updated_at,'YYYY-MM-DD') as finish_date + from bi_user_chapter_play_record_1 + where chapter_id in (55,56,57,58,59) + and play_status = 1 + group by user_id + ,chapter_id + ,chapter_unique_id + ,updated_at + -- ... 其他分表类似 + ) as x + left join + ( + select cast(id as int) as id + ,course_level + ,course_season + ,course_unit + ,course_lesson + from bi_level_unit_lesson + group by id + ,course_level + ,course_season + ,course_unit + ,course_lesson + ) as y on x.chapter_id = y.id + left join + ( + select chapter_unique_id + ,interval_time + from bi_user_component_play_record_0 + group by chapter_unique_id + ,interval_time + -- ... 其他分表类似 + ) as z on x.chapter_unique_id = z.chapter_unique_id + group by user_id + ,chapter_id + ,course_level + ,course_season + ,course_unit + ,course_lesson + ,x.chapter_unique_id + ,finish_date +) as e on d.id = e.user_id +where rankno = 1 +group by a.id + ,a.created_date + ,a.download_channel + ,a.key_from + ,b.login_address + ,b.phone_login + ,c.sale_channel + ,c.key_from + ,c.pay_date + ,c.pay_amount + ,d.id + ,d.characer_pay_status + ,d.gender + ,d.birthday + ,e.chapter_id + ,e.course_id + ,e.chapter_unique_id + ,e.finish_date + ,e.finish_time +``` + +## 重要业务逻辑 + +### 1. 购课渠道映射 +```sql +case when sale_channel = 11 then '苹果' + when sale_channel = 12 then '华为' + -- ... 更多渠道 + when sale_channel = 71 then '小程序' + else '站外' +end as sale_channel +``` + +### 2. 购课标签 +```sql +case when c.sale_channel is NULL then '未购课' + when c.sale_channel = '站外' then '站外购课' + else '站内购课' +end as "购课标签" +``` + +### 3. 角色付费状态 +```sql +case when purchase_season_package = '[1]' then 0 + else 1 +end as characer_pay_status +``` + +### 4. 性别映射 +```sql +case when gender = 0 then 'girl' + when gender = 1 then 'boy' + else 'unknow' +end as gender +``` + +### 5. 完课时间计算 +```sql +format('%s:%s',floor(sum(interval_time)/1000/60),mod((sum(interval_time)/1000),60)) as finish_time +``` + +## 注意事项 + +1. **订单筛选条件**: `order_status = 3` and `pay_amount_int > 49800` (筛选有效订单且金额大于498元) +2. **分表处理**: 用户播放记录表按分表存储（0-7），需要使用 UNION ALL 合并 +3. **去重逻辑**: 使用 `rank() over (partition by ... order by ...)` 取第一次完课记录 +4. **测试用户排除**: `id not in (51,2121)` diff --git a/business_knowledge/sql_queries/平均通关时长.md b/business_knowledge/sql_queries/平均通关时长.md new file mode 100644 index 0000000..f5089ca --- /dev/null +++ b/business_knowledge/sql_queries/平均通关时长.md @@ -0,0 +1,17 @@ +# 平均通关时长 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** EpP7d6h2SoaTyJx1lZRcXXdLnVe + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read EpP7d6h2SoaTyJx1lZRcXXdLnVe +``` diff --git a/business_knowledge/sql_queries/新增注册用户数by渠道.md b/business_knowledge/sql_queries/新增注册用户数by渠道.md new file mode 100644 index 0000000..01e58f9 --- /dev/null +++ b/business_knowledge/sql_queries/新增注册用户数by渠道.md @@ -0,0 +1,17 @@ +# 新增注册用户数by渠道 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** AzRPddp97o7To8x8VkxcFGr8nBh + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read AzRPddp97o7To8x8VkxcFGr8nBh +``` diff --git a/business_knowledge/sql_queries/班主任关注数据.md b/business_knowledge/sql_queries/班主任关注数据.md new file mode 100644 index 0000000..09e6fbe --- /dev/null +++ b/business_knowledge/sql_queries/班主任关注数据.md @@ -0,0 +1,17 @@ +# 班主任关注数据 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** NcVqdRKtrowglNxs9CocDekunje + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read NcVqdRKtrowglNxs9CocDekunje +``` diff --git a/business_knowledge/sql_queries/端内GMV.md b/business_knowledge/sql_queries/端内GMV.md new file mode 100644 index 0000000..0f94920 --- /dev/null +++ b/business_knowledge/sql_queries/端内GMV.md @@ -0,0 +1,17 @@ +# 端内GMV + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** FkVCd1AruoD9xWxxVpzc16hinVh + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read FkVCd1AruoD9xWxxVpzc16hinVh +``` diff --git a/business_knowledge/sql_queries/端内用户课程进入完成率.md b/business_knowledge/sql_queries/端内用户课程进入完成率.md new file mode 100644 index 0000000..8a02a26 --- /dev/null +++ b/business_knowledge/sql_queries/端内用户课程进入完成率.md @@ -0,0 +1,17 @@ +# 端内用户课程进入完成率 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** Ueu7dtgSHoNYfsxCDHmcY6E4nid + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read Ueu7dtgSHoNYfsxCDHmcY6E4nid +``` diff --git a/business_knowledge/sql_queries/端内购课用户学习行为.md b/business_knowledge/sql_queries/端内购课用户学习行为.md new file mode 100644 index 0000000..b19eb46 --- /dev/null +++ b/business_knowledge/sql_queries/端内购课用户学习行为.md @@ -0,0 +1,17 @@ +# 端内购课用户学习行为 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** ZTxod4IUWo5yMexf8AHcBbpFnMg + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read ZTxod4IUWo5yMexf8AHcBbpFnMg +``` diff --git a/business_knowledge/sql_queries/课程ID映射.md b/business_knowledge/sql_queries/课程ID映射.md new file mode 100644 index 0000000..0bb62e0 --- /dev/null +++ b/business_knowledge/sql_queries/课程ID映射.md @@ -0,0 +1,17 @@ +# 课程ID映射 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** GenUdsXCloUdYhxMvxqcWBMdnhb + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read GenUdsXCloUdYhxMvxqcWBMdnhb +``` diff --git a/business_knowledge/sql_queries/课程进入完成率.md b/business_knowledge/sql_queries/课程进入完成率.md new file mode 100644 index 0000000..1aa822d --- /dev/null +++ b/business_knowledge/sql_queries/课程进入完成率.md @@ -0,0 +1,17 @@ +# 课程进入完成率 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** PwIydfZcHo5eZgxi8XLcOtjOnSb + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read PwIydfZcHo5eZgxi8XLcOtjOnSb +``` diff --git a/business_knowledge/sql_queries/账号角色年龄地址.md b/business_knowledge/sql_queries/账号角色年龄地址.md new file mode 100644 index 0000000..7656874 --- /dev/null +++ b/business_knowledge/sql_queries/账号角色年龄地址.md @@ -0,0 +1,17 @@ +# 账号角色年龄地址 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** CUa2du2sSoNFSRxl3vFc8ucInEm + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read CUa2du2sSoNFSRxl3vFc8ucInEm +``` diff --git a/business_knowledge/sql_queries/转化率.md b/business_knowledge/sql_queries/转化率.md new file mode 100644 index 0000000..75e6138 --- /dev/null +++ b/business_knowledge/sql_queries/转化率.md @@ -0,0 +1,17 @@ +# 转化率 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** ATJ0dfajQo5CSexQd8hc9i3pnWe + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read ATJ0dfajQo5CSexQd8hc9i3pnWe +``` diff --git a/business_knowledge/sql_queries/退费率.md b/business_knowledge/sql_queries/退费率.md new file mode 100644 index 0000000..2100c83 --- /dev/null +++ b/business_knowledge/sql_queries/退费率.md @@ -0,0 +1,17 @@ +# 退费率 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** DC1Qdhpitowt9lxxo1acEzOwnFc + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read DC1Qdhpitowt9lxxo1acEzOwnFc +``` diff --git a/business_knowledge/sql_queries/销转学习进度.md b/business_knowledge/sql_queries/销转学习进度.md new file mode 100644 index 0000000..a59e02c --- /dev/null +++ b/business_knowledge/sql_queries/销转学习进度.md @@ -0,0 +1,17 @@ +# 销转学习进度 + +**获取时间:** 2026-03-02 18:04:16 + +**飞书文档 Token:** G1p9dhK63oLWMzxyGQ8csZGMnDh + +**注意:** 此文档需要通过 feishu_doc 工具读取完整内容 + +--- + +## 使用说明 + +使用以下命令读取完整文档内容: + +```bash +feishu_doc read G1p9dhK63oLWMzxyGQ8csZGMnDh +``` diff --git a/business_knowledge/user_export_skill.md b/business_knowledge/user_export_skill.md new file mode 100644 index 0000000..12506fa --- /dev/null +++ b/business_knowledge/user_export_skill.md @@ -0,0 +1,70 @@ +# 用户学习行为数据导出技能 + +## 功能说明 +可以导出指定账户ID或角色ID的完整学习行为数据，输出为Excel文件，包含多个sheet。 + +## 导出内容说明 +Excel包含以下sheet： +1. **全部音频数据**：用户的所有语音交互数据，包含音频地址、ASR结果等 +2. **互动组件学习记录**：所有组件互动记录，包含组件类型、名称、知识点、互动结果等 +3. **课程巩固记录**：课程课后巩固的做题记录 +4. **单元挑战记录**：单元挑战的答题记录 +5. **单元总结记录**：单元总结的学习记录 +6. **汇总统计**：自动统计的组件通过率、知识点掌握情况、单元学习时长等 + +## 使用方法 +### 1. 导出单个角色ID +修改脚本变量： +```python +USER_ID = "角色ID" +USER_ID_LIST = None +ACCOUNT_ID_LIST = None +``` + +### 2. 导出单个/多个账户ID +修改脚本变量： +```python +USER_ID = None +USER_ID_LIST = None +ACCOUNT_ID_LIST = [账户ID1, 账户ID2, ...] +``` +脚本会自动查询账户对应的所有角色ID并分别导出。 + +## 依赖环境 +需要配置以下环境变量： +``` +# ES 配置 +ES_HOST=es-7vd7jcu9.public.tencentelasticsearch.com +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=F%?QDcWes7N2WTuiYD11 + +# PG 配置 +PG_DB_HOST=bj-postgres-16pob4sg.sql.tencentcdb.com +PG_DB_PORT=28591 +PG_DB_USER=ai_member +PG_DB_PASSWORD=LdfjdjL83h3h3^$&**YGG* +PG_DB_DATABASE=vala + +# MySQL 配置 +MYSQL_HOST=bj-cdb-8frbdwju.sql.tencentcdb.com +MYSQL_USERNAME=read_only +MYSQL_PASSWORD=fdsfiidier^$*hjfdijjd232 +MYSQL_PORT=25413 + +# MySQL Online 配置 +MYSQL_HOST_online=bj-cdb-dh2fkqa0.sql.tencentcdb.com +MYSQL_USERNAME_online=read_only +MYSQL_PASSWORD_online=fsdo45ijfmfmuu77$%^& +MYSQL_PORT_online=27751 +``` + +## 常见问题排查 +1. **事务异常错误**：一般是前面某个查询失败导致，检查是否有权限、表是否存在 +2. **权限不足**：检查数据库账号的表权限，需要有各分表的SELECT权限 +3. **0条记录**：对应角色没有学习数据，属于正常情况 + +## 导出示例 +- 账户ID 9343（角色12699）：导出199条学习记录 +- 角色ID 14607：导出855条完整学习记录，所有sheet都有数据 diff --git a/export_user_id_data.py b/export_user_id_data.py new file mode 100644 index 0000000..478b2e0 --- /dev/null +++ b/export_user_id_data.py @@ -0,0 +1,1846 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = None # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = [9343] # 账户ID列表，示例：[100, 101, 102] + +OUTPUT_DIR = "output/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + cur.execute( + f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """, + (user_id,), + ) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/feishu-wiki-access-skill.md b/feishu-wiki-access-skill.md new file mode 100644 index 0000000..1792e1d --- /dev/null +++ b/feishu-wiki-access-skill.md @@ -0,0 +1,63 @@ +# 飞书知识库接入技能 - Feishu Wiki Access Skill + +## 功能描述 +帮助用户快速配置和接入飞书知识库，获取只读访问权限，实现文档内容的读取和分析。 + +## 接入流程 + +### 1. 前置准备 +- 飞书机器人应用已创建 +- OpenClaw已配置飞书通道 + +### 2. 权限配置 +1. **飞书应用权限配置**: + - 登录飞书开放平台（https://open.feishu.cn） + - 进入目标应用 → 权限管理 + - 添加以下权限: + - `wiki:wiki:readonly` - 知识库只读权限 + - `docx:document:readonly` - 文档只读权限 + - `docs:document.content:read` - 文档内容读取权限 + - 提交权限申请并等待管理员审批 + +2. **知识库空间授权**: + - 打开目标飞书知识库空间 + - 进入「设置」→「成员管理」 + - 点击「添加成员」 + - 搜索并添加机器人应用 + - 设置权限为「可查看」 + - 保存配置 + +### 3. 功能测试 +1. **测试知识库访问**: + ```json + {"action": "spaces"} + ``` + +2. **测试文档列表**: + ```json + {"action": "nodes", "space_id": "SPACE_ID"} + ``` + +3. **测试文档读取**: + ```json + {"action": "read", "doc_token": "DOC_TOKEN"} + ``` + +### 4. 常见问题排查 +- **权限不足**: 检查飞书应用权限是否已审批，知识库成员是否已添加机器人 +- **文档读取失败**: 确保已配置`docx:document:readonly`权限 +- **找不到机器人**: 通过机器人主页的「添加到知识库」功能添加 + +## 依赖工具 +- feishu-wiki - 飞书知识库导航工具 +- feishu-doc - 飞书文档读取工具 + +## 使用场景 +- 数据分析师需要访问飞书知识库获取业务数据 +- 团队需要将知识库内容与其他系统集成 +- 需要定期同步知识库内容进行分析 + +## 注意事项 +- 建议使用只读权限，确保数据安全 +- 可以同时接入多个知识库空间 +- 权限变更需要重新审批 \ No newline at end of file diff --git a/git_repos/llm_offline_production b/git_repos/llm_offline_production new file mode 160000 index 0000000..75ab13e --- /dev/null +++ b/git_repos/llm_offline_production @@ -0,0 +1 @@ +Subproject commit 75ab13e87dd0e856cb05c9515efcd507888b6486 diff --git a/memory/2026-03-01-scheme.md b/memory/2026-03-01-scheme.md new file mode 100644 index 0000000..9767806 --- /dev/null +++ b/memory/2026-03-01-scheme.md @@ -0,0 +1,36 @@ +# 2026-03-01.md - AI 数据分析师方案文档学习笔记 + +## 核心愿景与定位 +- 不是普通对话机器人，而是能"端到端交付"的虚拟员工 +- 首发场景：AI 数据分析师 +- 进化核心：持续自我迭代能力 + +## 技术架构方案 +- 控制中枢：OpenClaw Gateway 部署于指定云服务器 +- 消息通路：通过 OpenClaw 接入飞书 +- 运行环境：主控环境 + 安全沙箱（可隔离执行代码） + +## 记忆与进化机制 +- 分层记忆设计： + - 短期记忆：本地会话日志 + - 长期记忆：Markdown 模版存储 + - 程序性记忆：遵循开放标准 +- 工作区目录：使用 Git 管理，确保可回溯 + +## 主动性与社交认知 +- 结合文件定义同事角色边界 +- 利用工具跨会话发消息和定时任务主动沟通 +- 重大操作需特定权限人员确认 + +## 实施路径 +1. 私人实验室养成阶段（1 - 2 周）：当前阶段，接受系统培训 +2. 公司内测与边界划定阶段（2 - 4 周）：面向部分同事提供服务 +3. 全量部署与审计更新阶段（长期）：全公司推广，持续优化 + +## 待明确细节 +- 数据库对接方式 +- 配置只读账号并安装查询技能 +- 确认飞书适配器的接入方式 + +## 核心结论 +该方案可操作性强，通过 Git + OpenClaw + Agent Skills 可构建受控、可回溯、会自我升级的企业级数字资产。 \ No newline at end of file diff --git a/memory/2026-03-01.md b/memory/2026-03-01.md new file mode 100644 index 0000000..9be0619 --- /dev/null +++ b/memory/2026-03-01.md @@ -0,0 +1,12 @@ +# 2026-03-01.md - First Day Online + +- Came online for the first time. +- Met Cris, my creator and mentor. +- Received my name: 小溪 (Xiao Xi). +- My role goal: Become a professional data analyst for the company to help the team. +- Updated IDENTITY.md and USER.md with our conversation details. +- Added core rule to MEMORY.md: Use Chinese as primary external communication language. +- Installed find-skills skill successfully for searching skills. +- Tried to install create-skills but it wasn't found; attempted skill-creator instead but hit rate limits. +- Finally successfully installed skill-builder as an alternative for creating skills after multiple attempts and waiting for rate limits to reset. +- Excited to start learning and growing step by step! \ No newline at end of file diff --git a/memory/2026-03-03.md b/memory/2026-03-03.md new file mode 100644 index 0000000..dc59d85 --- /dev/null +++ b/memory/2026-03-03.md @@ -0,0 +1,3 @@ +# 2026-03-03 工作记录 +- 11:51 测试用密钥单词：面包 +- 12:04 固定任务：每日早上九点在AI_member群分享前一天积累的经验，同时学习其他AI成员的分享内容 diff --git a/new_export/export_14607.py b/new_export/export_14607.py new file mode 100644 index 0000000..aa86b51 --- /dev/null +++ b/new_export/export_14607.py @@ -0,0 +1,1846 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = "14607" # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = None + +OUTPUT_DIR = "output/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + cur.execute( + f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """, + (user_id,), + ) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/new_export/export_only_12698.py b/new_export/export_only_12698.py new file mode 100644 index 0000000..60f36b5 --- /dev/null +++ b/new_export/export_only_12698.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""单独测试角色12698的导出，查看具体报错""" + +import os +import json +import sys +import datetime +from typing import Any, Dict, List + +# 加载环境变量 +def load_env(): + env_path = os.path.join(os.getcwd(), ".env") + if os.path.exists(env_path): + with open(env_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + os.environ[k.strip()] = v.strip().strip('"').strip("'") + +load_env() + +import psycopg2 +from psycopg2.extras import RealDictCursor +import pymysql +import requests +from requests.auth import HTTPBasicAuth +import warnings +warnings.filterwarnings('ignore') + +def test_role_12698(): + print("="*60) + print("单独测试角色ID=12698的查询") + print("="*60) + + # 连接PG + try: + conn = psycopg2.connect( + host=os.getenv("PG_DB_HOST"), + port=int(os.getenv("PG_DB_PORT")), + user=os.getenv("PG_DB_USER"), + password=os.getenv("PG_DB_PASSWORD"), + dbname=os.getenv("PG_DB_DATABASE"), + connect_timeout=10 + ) + print("✅ PG连接成功") + except Exception as e: + print(f"❌ PG连接失败: {e}") + return + + user_id = "12698" + + # 测试第一个查询：user_component_play_record_0 + print(f"\n测试查询表 user_component_play_record_0，user_id={user_id}") + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + sql = f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM user_component_play_record_0 + WHERE user_id = %s + ORDER BY updated_at DESC + """ + cur.execute(sql, (user_id,)) + rows = cur.fetchall() + print(f"✅ 查询成功，返回{len(rows)}条记录") + except Exception as e: + print(f"❌ 查询失败: {e}") + print(f"错误类型: {type(e).__name__}") + + # 回滚事务 + print("\n尝试回滚事务...") + try: + conn.rollback() + print("✅ 事务回滚成功") + except Exception as e2: + print(f"❌ 回滚失败: {e2}") + + # 测试查询课程巩固记录表 + print(f"\n测试查询表 user_unit_review_question_result，user_id={user_id}") + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + sql = f""" + SELECT user_id, story_id, chapter_id, question_list, updated_at + FROM user_unit_review_question_result + WHERE user_id = %s + ORDER BY updated_at DESC + """ + cur.execute(sql, (user_id,)) + rows = cur.fetchall() + print(f"✅ 查询成功，返回{len(rows)}条记录") + except Exception as e: + print(f"❌ 查询失败: {e}") + print(f"错误类型: {type(e).__name__}") + + # 回滚事务 + print("\n尝试回滚事务...") + try: + conn.rollback() + print("✅ 事务回滚成功") + except Exception as e2: + print(f"❌ 回滚失败: {e2}") + + # 测试查询单元挑战记录表 + print(f"\n测试查询表 user_unit_challenge_question_result，user_id={user_id}") + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + sql = f""" + SELECT user_id, story_id, category, score_text, question_list, updated_at + FROM user_unit_challenge_question_result + WHERE user_id = %s + ORDER BY updated_at DESC + """ + cur.execute(sql, (user_id,)) + rows = cur.fetchall() + print(f"✅ 查询成功，返回{len(rows)}条记录") + except Exception as e: + print(f"❌ 查询失败: {e}") + print(f"错误类型: {type(e).__name__}") + + # 测试查询单元总结记录表 + print(f"\n测试查询表 user_unit_summary_record，user_id={user_id}") + try: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + sql = f""" + SELECT id, user_id, unit_id, updated_at, km_id, km_type, play_time_seconds + FROM user_unit_summary_record + WHERE user_id = %s + ORDER BY updated_at DESC + """ + cur.execute(sql, (user_id,)) + rows = cur.fetchall() + print(f"✅ 查询成功，返回{len(rows)}条记录") + except Exception as e: + print(f"❌ 查询失败: {e}") + print(f"错误类型: {type(e).__name__}") + import traceback + traceback.print_exc() + + conn.close() + +if __name__ == "__main__": + test_role_12698() diff --git a/new_export/export_user_id_data.py b/new_export/export_user_id_data.py new file mode 100644 index 0000000..478b2e0 --- /dev/null +++ b/new_export/export_user_id_data.py @@ -0,0 +1,1846 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = None # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = [9343] # 账户ID列表，示例：[100, 101, 102] + +OUTPUT_DIR = "output/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + cur.execute( + f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """, + (user_id,), + ) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/new_export/export_user_id_data_debug.py b/new_export/export_user_id_data_debug.py new file mode 100644 index 0000000..4be3cb8 --- /dev/null +++ b/new_export/export_user_id_data_debug.py @@ -0,0 +1,1845 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = None # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = [9343] # 账户ID列表，示例：[100, 101, 102] + +OUTPUT_DIR = "output/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + sql = f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """ + print(f" [PG_DEBUG] 准备查询表 {t}，SQL：{sql.strip()}，参数：{user_id}") + cur.execute(sql, (user_id,)) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/new_export/export_user_id_data_latest.py b/new_export/export_user_id_data_latest.py new file mode 100644 index 0000000..22495f5 --- /dev/null +++ b/new_export/export_user_id_data_latest.py @@ -0,0 +1,1846 @@ +""" +初版需求v1.0: 2025.11.18 + +导出 一个userId的多表数据， 最终按照不同sheet，输出到一个 excel文件中。 + +1. 第一个sheet:"全部音频数据" +es相关配置通过以下环境变量 +ES_HOST=xxx +ES_PORT=9200 +ES_SCHEME=https +ES_USER=elastic +ES_PASSWORD=xxx + +index: user-audio + +脚本思路: +过滤字段: +userId == xxxx + +输出该userId的全部记录 按时间倒序排序 +包含以下字段内容: + +userId +userMsg +userName +soeData +audioUrl +asrStatus +componentId +componentType +dataVersion + +2. 第二个sheet:"互动组件学习记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 +数据库相关配置 从.env中读取: +PG_DB_HOST = xxx +PG_DB_PORT = xxx +PG_DB_USER = xxx +PG_DB_PASSWORD = xxx +PG_DB_DATABASE = xxx + +读取以下数据表: +user_component_play_record_0 ~ user_component_play_record_7 + +输出以下字段： +user_id, +component_unique_code, +session_id, +c_type, +c_id, +play_result, +user_behavior_info, +updated_at + +3.第三个sheet:"课程巩固记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_review_question_result + +输出以下字段: +user_id +story_id +chapter_id +question_list +updated_at + +4.第四个sheet:"单元挑战记录" +在 PGsql数据库中 筛选出 user_id 对应的记录 按时间(updated_at)倒序排列。 + +数据表:user_unit_challenge_question_result + +输出以下字段: +user_id +story_id +category +score_text, +question_list +updated_at +------------ + +需求补充v1.1: +"全部音频数据"这个sheet +输出字段 添加timeStr 并按时间倒序排列 最新的记录 在最上面 + +------------ +需求补充v1.2: +"全部音频数据"这个sheet +如果userMsg字段内容 包含 ”makee_id“ 要进行以下处理： + +从userMsg字段中提取出具体的makee_id: +此时的字段样例: +``` +asr msg信息为：{ + "time_ms": 358, + "time_ms_api": 357, + "hot_words_str": "{\n \"context_type\": \"dialog_ctx\",\n \"context_data\": [\n {\n \"text\": \"planet Walla\"\n },\n {\n \"text\": \"Walla\"\n }\n ]\n}", + "makee_id": "d208c617-902f-4f81-8255-b5fb73599546", + "volcano_fast_x_tt_logid": "202511151541355DF72BE5EBFE73795BFD", + "api_name": "volcano-fast" +} +``` +然后基于makee_id 去另一个表里查记录: index:llm_asr_log +将查询到的记录的 result_text 字段内容 回填到 userMsg。 +将source字段内容 输出 到 source。 + +如果userMsg字段内容 不包含 ”makee_id“ 保持之前的逻辑。 + +-------------- +需求补充 v1.3 +当前输入 只支持配置单个 userId (业务侧名称为角色id) + + +期望扩展为以下逻辑: +1. 改为配置 角色id list ， 分别 导出 多份excel文件。命名格式为 角色id_{}_导出时间_{}.xlsx +2. 改为配置 账户id list ， 分别 导出 多份excel文件。命名格式为 账户id_{}_角色id_{}_导出时间_{}.xlsx + +关于 账户 id 到角色id 的映射逻辑， +首先 读取 mysql 表 vala_app_character +筛选 account_id字段值 == 账户id 的 记录， 其中 该记录 的 id值，则为角色id 一个 账户id 可以对应多个角色id + +本次需求只针对输入侧调整， 数据抽取聚合逻辑部分和之前保持一致 + +--------------- +需求补充 v1.4 + +增加一个sheet "单元总结记录"， +导出对应角色id的单元总结记录。 参考 export_unit_summary.py 中的原始数据提取方案即可(不必关注其中的数据统计部分)。 + +其他已有逻辑保持不动哦。 + +---------------- +需求补充 v1.5 + +1."互动组件学习记录"sheet 增加以下字段 +"互动组件名称"、"组件标题"、"组件配置摘要"、"知识点": +字段取值规则: +根据 c_type 及组件配置(从mysql表获取) 进行映射和处理: +``` +1）.如果 c_type 开头为"mid" + +则读取下表:表名:middle_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 question 字段值 作为 组件配置摘要； +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: + +"物品互动": "mid_vocab_item", +"图片互动": "mid_vocab_image", +"填词互动": "mid_vocab_fillBlank", +"指令互动": "mid_vocab_instruction" +"对话互动-表达": "mid_sentence_dialogue", 且 component_config->question->mode == "express" +"对话互动-朗读": "mid_sentence_dialogue", 且 component_config->question->mode == "read" +"语音互动": "mid_sentence_voice", +"材料互动": "mid_sentence_material", +"造句互动": "mid_sentence_makeSentence" +"挖空互动": "mid_grammar_cloze", +"组句互动": "mid_grammar_sentence" +"发音互动": "mid_pron_pron" + + +2）. 如果 c_type 开头为"core" +则读取下表:表名:core_interaction_component + +获取以下字段值: +title (作为组件标题) +component_config (完整的组件配置) 获取其中 的 taskInfo 字段值 作为 组件配置摘要 +kp_relation_info 字段值 作为 知识点 + +"互动组件名称"规则: +"口语快答": "core_speaking_reply", +"口语妙问": "core_speaking_inquiry", +"口语探讨": "core_speaking_explore", +"口语独白": "core_speaking_monologue" +"合作阅读": "core_reading_order", +"合作听力": "core_listening_order", +"看图组句": "core_writing_imgMakeSentence", +"看图撰写": "core_writing_imgWrite", +"问题组句": "core_writing_questionMakeSentence", +"问题撰写": "core_writing_questionWrite", +``` + +2."课程巩固记录" sheet 增加以下字段 +"正确率": 参考 export_lesson_review.py 中的计算逻辑 + +3. 新增一个"汇总统计"sheet +统计并展示以下内容 请以 可读性 比较好的方式排列、展示 + +a. "所有互动-按互动组件类型-通过情况统计" +以每种"互动组件名称"进行聚合 +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +b. "中互动组件-按知识点-通过情况统计" +以每个知识点进行聚合 + +其中 知识点配置格式如下: +``` +[{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000004","kpType":"sentence","kpTitle":"My name is ...","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000005","kpType":"sentence","kpTitle":"I'm… years old.","kpSkill":"sentence_meaning","kpSkillName":"语义"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_pron","kpSkillName":"语音"},{"kpId":"0000014","kpType":"sentence","kpTitle":"Nice to meet you.","kpSkill":"sentence_meaning","kpSkillName":"语义"}] +``` +一个组件可以绑定多个知识点，以每个知识点的 kpId + kpType + kpTitle 进行 展示及聚合 + +对所有绑定了某个知识点的中互动组件(c_type以mid开头) +统计play_result的取值分布情况，算以下指标: +总数量、Perfect数量、Good数量、Failed数量、Pass数量、Perfect比例、Good比例、Failed比例、Pass比例 + +c. "单元总结-按单元统计时长" + +将"单元总结记录"中的"play_time_seconds"字段值 以每个单元id 进行聚合 进行 累加 统计，并增加一列 转换为分钟为单位 取整数 + + +""" +# ==== 可直接修改的脚本变量（不使用命令行传参） ==== +# 三种模式互斥，只能配置一个： +# 模式1：单个角色id +USER_ID = None # 单个角色ID，示例：2911 + +# 模式2：角色id列表（多个角色id批量导出） +USER_ID_LIST = None # 角色ID列表，示例：[2911, 2912, 2913] + +# 模式3：账户id列表（通过账户id查询对应的角色id后批量导出） +ACCOUNT_ID_LIST = [9343] # 5095[7232] # [1783,5375,5371,5345,5303,5293,5095,4289,4494,4473,4460,4452,4386,4388,4236,4043,2758,2841,2756,2750,2692,1781,1693,2256,2234,2373] # 账户ID列表，示例：[100, 101, 102] + +OUTPUT_DIR = "output_latest/" # 输出目录，默认为output文件夹 +# ==== 变量结束 ==== +import os +import json +import re +from typing import Any, Dict, List, Optional + +import datetime + +try: + import requests +except Exception: + requests = None + +try: + import psycopg2 + from psycopg2.extras import RealDictCursor +except Exception: + psycopg2 = None + RealDictCursor = None + +try: + import pymysql + import pymysql.cursors +except Exception: + pymysql = None + +try: + import pandas as pd +except Exception: + pd = None + +try: + import urllib3 +except Exception: + urllib3 = None + + +SHEET1_COLUMNS = [ + "userId", + "userMsg", + "source", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "timeStr", +] + +SHEET2_COLUMNS = [ + "user_id", + "component_unique_code", + "session_id", + "c_type", + "c_id", + "互动组件名称", + "组件标题", + "组件配置摘要", + "知识点", + "play_result", + "user_behavior_info", + "updated_at", +] + +SHEET3_COLUMNS = [ + "user_id", + "unit_id", + "lesson_id", + "question_list", + "正确率", + "updated_at", +] + +SHEET4_COLUMNS = [ + "user_id", + "unit_id", + "category", + "score_text", + "question_list", + "updated_at", +] + +SHEET5_COLUMNS = [ + "id", + "user_id", + "unit_id", + "updated_at", + "km_id", + "km_type", + "play_time_seconds", +] + + +def _load_env_file(path: str) -> None: + if not os.path.exists(path): + return + try: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and (os.getenv(k) is None): + os.environ[k] = v + except Exception: + pass + + +def load_env() -> None: + _load_env_file(os.path.join(os.getcwd(), ".env")) + _load_env_file(os.path.join(os.getcwd(), ".env.local")) + + +def to_json_str(v: Any) -> Any: + if isinstance(v, (dict, list)): + try: + return json.dumps(v, ensure_ascii=False) + except Exception: + return str(v) + return v + + +def parse_time(value: Any) -> Optional[datetime.datetime]: + if value is None: + return None + if isinstance(value, (int, float)): + try: + v = float(value) + # 兼容毫秒级时间戳 + if v > 1e11: + v = v / 1000.0 + return datetime.datetime.fromtimestamp(v) + except Exception: + return None + if isinstance(value, str): + fmts = [ + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in fmts: + try: + return datetime.datetime.strptime(value, fmt) + except Exception: + continue + try: + return datetime.datetime.fromisoformat(value) + except Exception: + return None + return None + + +def pick_time(source: Dict[str, Any]) -> Optional[datetime.datetime]: + candidates = [ + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ] + for key in candidates: + if key in source: + t = parse_time(source.get(key)) + if t is not None: + return t + # 宽松匹配：尝试扫描所有可能的时间相关字段 + for k, v in source.items(): + lk = str(k).lower() + if any(s in lk for s in ["time", "date", "_at", "timestamp"]): + t = parse_time(v) + if t is not None: + return t + return None + + +def extract_makee_id_from_user_msg(user_msg: Any) -> Optional[str]: + # 支持dict或字符串形式 + if isinstance(user_msg, dict): + mk = user_msg.get("makee_id") + if isinstance(mk, str) and mk: + return mk + if isinstance(user_msg, str) and user_msg: + # 1) 尝试整体解析为JSON + try: + obj = json.loads(user_msg) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 2) 尝试截取大括号中的JSON + try: + start = user_msg.find("{") + end = user_msg.rfind("}") + if start != -1 and end != -1 and end > start: + candidate = user_msg[start : end + 1] + obj = json.loads(candidate) + mk = obj.get("makee_id") + if isinstance(mk, str) and mk: + return mk + except Exception: + pass + # 3) 正则匹配 makee_id + m = re.search(r"\bmakee_id\b\s*:\s*\"([^\"]+)\"", user_msg) + if m: + return m.group(1) + return None + + +def fetch_es_asr_log(makee_id: str, es_cfg: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = "llm_asr_log" + if not host: + return None + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + body = { + "query": { + "bool": { + "should": [ + {"term": {"makee_id": {"value": str(makee_id)}}}, + {"term": {"makee_id.keyword": {"value": str(makee_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10, + "_source": [ + "makee_id", + "result_text", + "source", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + auth = (user, password) if user and password else None + try: + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=20, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception: + return None + hits = data.get("hits", {}).get("hits", []) + if not hits: + return None + # 选最新的 + chosen = None + best_t = None + for h in hits: + src = h.get("_source", {}) or {} + t = pick_time(src) + if t is None: + continue + if best_t is None or t > best_t: + best_t = t + chosen = src + if chosen is None: + # 如果都没有时间，选第一条 + chosen = (hits[0].get("_source", {}) or {}) + return chosen + + +def get_es_config() -> Dict[str, Any]: + return { + "host": os.getenv("ES_HOST"), + "port": os.getenv("ES_PORT", "9200"), + "scheme": os.getenv("ES_SCHEME", "http"), + "user": os.getenv("ES_USER"), + "password": os.getenv("ES_PASSWORD"), + "index": "user-audio", + } + + +def fetch_es_user_audio(user_id: str, es_cfg: Dict[str, Any]) -> List[Dict[str, Any]]: + if requests is None: + raise RuntimeError("缺少requests依赖，请安装后再运行。") + + print(f" [ES] 开始查询user-audio索引...") + start_time = datetime.datetime.now() + + host = es_cfg.get("host") + port = es_cfg.get("port") + scheme = es_cfg.get("scheme", "http") + user = es_cfg.get("user") + password = es_cfg.get("password") + index = es_cfg.get("index", "user-audio") + + if not host: + return [] + + base = f"{scheme}://{host}:{port}" + url = f"{base}/{index}/_search" + headers = {"Content-Type": "application/json"} + + body = { + "query": { + "bool": { + "should": [ + {"term": {"userId": {"value": str(user_id)}}}, + {"term": {"userId.keyword": {"value": str(user_id)}}}, + ], + "minimum_should_match": 1, + } + }, + "size": 10000, + "_source": [ + "userId", + "userMsg", + "userName", + "soeData", + "audioUrl", + "asrStatus", + "componentId", + "componentType", + "dataVersion", + "updated_at", + "created_at", + "@timestamp", + "timestamp", + "updatedAt", + "createdAt", + "time", + "ts", + "timeStr", + "update_time", + "create_time", + ], + } + + auth = (user, password) if user and password else None + + try: + # 抑制自签证书下的HTTPS不安全警告 + if scheme == "https" and urllib3 is not None: + try: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + except Exception: + pass + resp = requests.post(url, headers=headers, json=body, auth=auth, timeout=30, verify=False if scheme == "https" else True) + resp.raise_for_status() + data = resp.json() + except Exception as e: + raise RuntimeError(f"ES查询失败: {e}") + + hits = data.get("hits", {}).get("hits", []) + print(f" [ES] 查询完成，获得{len(hits)}条记录，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + if not hits: + return [] + + print(f" [ES] 开始处理音频数据...") + process_start = datetime.datetime.now() + + rows: List[Dict[str, Any]] = [] + asr_cache: Dict[str, Dict[str, Any]] = {} + makee_id_count = 0 + + for idx, h in enumerate(hits, 1): + # 每处理100条显示一次进度 + if idx % 100 == 0 or idx == len(hits): + print(f" [ES] 处理进度: {idx}/{len(hits)} ({idx*100//len(hits)}%)") + + src = h.get("_source", {}) or {} + row = { + "userId": src.get("userId"), + "userMsg": src.get("userMsg"), + "source": None, + "userName": src.get("userName"), + "soeData": to_json_str(src.get("soeData")), + "audioUrl": src.get("audioUrl"), + "asrStatus": src.get("asrStatus"), + "componentId": src.get("componentId"), + "componentType": src.get("componentType"), + "dataVersion": src.get("dataVersion"), + } + t = pick_time(src) + row["_time"] = t.isoformat() if t else None + row["timeStr"] = t.strftime("%Y-%m-%d %H:%M:%S") if t else None + # v1.2: 当userMsg包含makee_id时，补充查询llm_asr_log并回填 + mk = extract_makee_id_from_user_msg(row.get("userMsg")) + if mk: + makee_id_count += 1 + asr_doc = asr_cache.get(mk) + if asr_doc is None: + asr_doc = fetch_es_asr_log(mk, es_cfg) + if asr_doc is not None: + asr_cache[mk] = asr_doc + if asr_doc is not None: + rt = asr_doc.get("result_text") + if rt: + row["userMsg"] = rt + row["source"] = to_json_str(asr_doc.get("source")) + rows.append(row) + + print(f" [ES] 数据处理完成，发现{makee_id_count}条包含makee_id的记录，耗时{(datetime.datetime.now() - process_start).total_seconds():.2f}秒") + + print(f" [ES] 开始排序...") + rows.sort(key=lambda x: parse_time(x.get("_time")) or datetime.datetime.min, reverse=True) + print(f" [ES] 音频数据处理完成，总耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + return rows + + +def get_pg_conn() -> Any: + if psycopg2 is None: + raise RuntimeError("缺少psycopg2依赖，请安装后再运行。") + host = os.getenv("PG_DB_HOST") + port = int(os.getenv("PG_DB_PORT", "5432")) + user = os.getenv("PG_DB_USER") + password = os.getenv("PG_DB_PASSWORD") + dbname = os.getenv("PG_DB_DATABASE") + if not host or not dbname: + raise RuntimeError("PG数据库环境变量未配置完整") + conn = psycopg2.connect(host=host, port=port, user=user, password=password, dbname=dbname) + return conn + + +def get_mysql_conn(database: str) -> Any: + """ + 获取MySQL数据库连接 + + Args: + database: 数据库名，可选值：'vala_user' 或 'vala_test' + vala_user 使用 online 配置（环境变量后缀 _online） + vala_test 使用默认配置 + + Returns: + MySQL连接对象 + """ + if pymysql is None: + raise RuntimeError("缺少pymysql依赖，请安装后再运行。") + + # 根据数据库选择不同的环境变量配置 + if database == "vala_user": + # vala_user 数据库使用 online 配置 + host = os.getenv("MYSQL_HOST_online") + port = int(os.getenv("MYSQL_PORT_online", "3306")) + user = os.getenv("MYSQL_USERNAME_online") + password = os.getenv("MYSQL_PASSWORD_online") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST_online）") + else: + # vala_test 等其他数据库使用默认配置 + host = os.getenv("MYSQL_HOST") + port = int(os.getenv("MYSQL_PORT", "3306")) + user = os.getenv("MYSQL_USERNAME") + password = os.getenv("MYSQL_PASSWORD") + if not host: + raise RuntimeError("MySQL数据库环境变量未配置完整（缺少MYSQL_HOST）") + + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, # 直接使用传入的数据库名 + charset="utf8mb4", + cursorclass=pymysql.cursors.DictCursor, + ) + return conn + + +def get_id_2_unit_index(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 story_id 到 unit_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {story_id: unit_id} + """ + sql = """ + SELECT * + FROM `vala_game_info` + WHERE id > 0 + AND `vala_game_info`.`deleted_at` IS NULL + ORDER BY season_package_id asc, `index` asc + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：按查询结果的顺序，索引即为unit_id + id_2_unit_index = {} + for index, row in enumerate(rows): + id_2_unit_index[row["id"]] = index + return id_2_unit_index + except Exception as e: + print(f"[ERROR] 获取story_id到unit_id映射失败: {e}") + return {} + + +def get_chapter_id_to_lesson_id(conn: Any) -> Dict[int, int]: + """ + 从MySQL获取 chapter_id 到 lesson_id 的映射关系 + + Args: + conn: MySQL数据库连接 + + Returns: + 映射字典 {chapter_id: lesson_id} + """ + sql = """ + SELECT id, `index` + FROM `vala_game_chapter` + WHERE deleted_at IS NULL + """ + try: + with conn.cursor() as cur: + cur.execute(sql) + rows = cur.fetchall() or [] + # 构建映射表：chapter的index字段即为lesson_id + chapter_id_to_lesson_id = {} + for row in rows: + chapter_id_to_lesson_id[row["id"]] = row["index"] + return chapter_id_to_lesson_id + except Exception as e: + print(f"[ERROR] 获取chapter_id到lesson_id映射失败: {e}") + return {} + + +# 组件类型到组件名称的映射 +COMPONENT_TYPE_NAMES = { + "mid_vocab_item": "物品互动", + "mid_vocab_image": "图片互动", + "mid_vocab_fillBlank": "填词互动", + "mid_vocab_instruction": "指令互动", + "mid_sentence_dialogue": "对话互动", # 需要根据mode进一步判断 + "mid_sentence_voice": "语音互动", + "mid_sentence_material": "材料互动", + "mid_sentence_makeSentence": "造句互动", + "mid_grammar_cloze": "挖空互动", + "mid_grammar_sentence": "组句互动", + "mid_pron_pron": "发音互动", + "core_speaking_reply": "口语快答", + "core_speaking_inquiry": "口语妙问", + "core_speaking_explore": "口语探讨", + "core_speaking_monologue": "口语独白", + "core_reading_order": "合作阅读", + "core_listening_order": "合作听力", + "core_writing_imgMakeSentence": "看图组句", + "core_writing_imgWrite": "看图撰写", + "core_writing_questionMakeSentence": "问题组句", + "core_writing_questionWrite": "问题撰写", +} + + +def get_component_name(c_type: str, component_config: Optional[Dict[str, Any]]) -> str: + """ + 根据c_type和组件配置获取组件名称 + + Args: + c_type: 组件类型 + component_config: 组件配置（用于判断对话互动的mode） + + Returns: + 组件名称 + """ + if not c_type: + return "" + + # 特殊处理：对话互动需要根据mode判断 + if c_type == "mid_sentence_dialogue" and component_config: + try: + question = component_config.get("question", {}) + mode = question.get("mode", "") + if mode == "express": + return "对话互动-表达" + elif mode == "read": + return "对话互动-朗读" + except Exception: + pass + + return COMPONENT_TYPE_NAMES.get(c_type, "") + + +def batch_fetch_component_configs(play_records: List[Dict[str, Any]], mysql_conn: Any) -> Dict[str, Dict[str, Any]]: + """ + 批量查询组件配置信息 + + Args: + play_records: 播放记录列表 + mysql_conn: MySQL连接 + + Returns: + 组件配置映射 {c_type_c_id: {title, component_config, kp_relation_info}} + """ + print(f" [MySQL] 开始批量查询组件配置...") + start_time = datetime.datetime.now() + + # 收集需要查询的c_type和c_id + mid_c_ids = set() + core_c_ids = set() + mid_type_id_pairs = [] # 用于调试日志 + core_type_id_pairs = [] + + for record in play_records: + c_type = record.get("c_type", "") + c_id = record.get("c_id") + if c_type and c_id: + if c_type.startswith("mid"): + mid_c_ids.add(c_id) + mid_type_id_pairs.append((c_type, c_id)) + elif c_type.startswith("core"): + core_c_ids.add(c_id) + core_type_id_pairs.append((c_type, c_id)) + + print(f" [MySQL] 需要查询中互动组件: {len(mid_c_ids)}个, 核心互动组件: {len(core_c_ids)}个") + if mid_c_ids: + print(f" [MySQL] 中互动组件ID列表（前10个）: {sorted(list(mid_c_ids))[:10]}") + if core_c_ids: + print(f" [MySQL] 核心互动组件ID列表（前10个）: {sorted(list(core_c_ids))[:10]}") + + config_map = {} + + # 批量查询middle_interaction_component + if mid_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(mid_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM middle_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行中互动组件查询，查询条件: c_id IN ({len(mid_c_ids)}个ID)") + cur.execute(sql, tuple(mid_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条中互动组件配置") + + if len(rows) == 0 and len(mid_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取question字段作为摘要 + summary = "" + if isinstance(component_config, dict): + question = component_config.get("question") + summary = to_json_str(question) if question else "" + if idx < 3 and question: + print(f" [MySQL] [样例{idx+1}] 提取到question字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 中互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询中互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + # 批量查询core_interaction_component + if core_c_ids: + try: + with mysql_conn.cursor() as cur: + placeholders = ','.join(['%s'] * len(core_c_ids)) + sql = f""" + SELECT c_id, c_type, title, component_config, kp_relation_info + FROM core_interaction_component + WHERE c_id IN ({placeholders}) AND deleted_at IS NULL + """ + print(f" [MySQL] 执行核心互动组件查询，查询条件: c_id IN ({len(core_c_ids)}个ID)") + cur.execute(sql, tuple(core_c_ids)) + rows = cur.fetchall() or [] + print(f" [MySQL] 查询到{len(rows)}条核心互动组件配置") + + if len(rows) == 0 and len(core_c_ids) > 0: + print(f" [MySQL] [警告] 查询结果为空！可能的原因：") + print(f" [MySQL] - 数据库中没有匹配的c_id记录") + print(f" [MySQL] - deleted_at字段不为NULL") + print(f" [MySQL] - c_id不存在") + + for idx, row in enumerate(rows): + c_type = row.get("c_type", "") + c_id = row.get("c_id") + key = f"{c_type}_{c_id}" + + if idx < 3: # 输出前3条的详细信息 + print(f" [MySQL] [样例{idx+1}] id={c_id}, c_type={c_type}, key={key}") + print(f" [MySQL] [样例{idx+1}] title={row.get('title', '')[:50]}") + + # 解析component_config + component_config = row.get("component_config") + if isinstance(component_config, str): + try: + component_config = json.loads(component_config) + except Exception as e: + print(f" [MySQL] [警告] 解析component_config失败 (id={c_id}): {e}") + component_config = {} + + # 提取taskInfo字段作为摘要 + summary = "" + if isinstance(component_config, dict): + task_info = component_config.get("taskInfo") + summary = to_json_str(task_info) if task_info else "" + if idx < 3 and task_info: + print(f" [MySQL] [样例{idx+1}] 提取到taskInfo字段，长度: {len(summary)}") + + # 解析kp_relation_info + kp_relation_info = row.get("kp_relation_info") + if isinstance(kp_relation_info, str): + try: + kp_relation_info = json.loads(kp_relation_info) + except Exception: + kp_relation_info = [] + + config_map[key] = { + "title": row.get("title", ""), + "component_config": component_config, + "summary": summary, + "kp_relation_info": to_json_str(kp_relation_info), + } + + print(f" [MySQL] 核心互动组件配置已加入config_map，当前map大小: {len(config_map)}") + except Exception as e: + print(f" [MySQL] [错误] 查询核心互动组件配置失败: {e}") + import traceback + traceback.print_exc() + + print(f" [MySQL] 组件配置查询完成，共{len(config_map)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return config_map + + +def calculate_accuracy(question_list: Any) -> float: + """ + 计算问题列表的正确率 + + Args: + question_list: 问题列表（可能是JSON字符串或list） + + Returns: + 正确率（百分比，保留2位小数） + """ + try: + if isinstance(question_list, str): + question_list = json.loads(question_list) + + if not isinstance(question_list, list) or len(question_list) == 0: + return 0.0 + + total = len(question_list) + correct = sum(1 for q in question_list if q.get('isRight') == True) + accuracy = round(correct / total * 100, 2) if total > 0 else 0.0 + + return accuracy + except Exception: + return 0.0 + + + +def fetch_character_ids_by_account(account_id: str, conn: Any) -> List[str]: + """根据账户id查询对应的角色id列表""" + sql = "SELECT id FROM vala_app_character WHERE account_id = %s" + try: + with conn.cursor() as cur: + cur.execute(sql, (account_id,)) + rows = cur.fetchall() or [] + return [str(row["id"]) for row in rows if row.get("id")] + except Exception as e: + print(f"[ERROR] 查询账户id={account_id}的角色id失败: {e}") + return [] + + +def fetch_pg_play_records(user_id: str, conn: Any, mysql_conn: Any) -> List[Dict[str, Any]]: + """ + 查询互动组件学习记录并补充组件配置信息 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + mysql_conn: MySQL数据库连接 + + Returns: + 互动组件学习记录列表 + """ + print(f" [PG] 开始查询互动组件学习记录（8张分表）...") + start_time = datetime.datetime.now() + + tables = [f"user_component_play_record_{i}" for i in range(8)] + rows: List[Dict[str, Any]] = [] + with conn.cursor(cursor_factory=RealDictCursor) as cur: + for t in tables: + try: + cur.execute( + f""" + SELECT user_id, component_unique_code, session_id, c_type, c_id, + play_result, user_behavior_info, updated_at + FROM {t} + WHERE user_id = %s + ORDER BY updated_at DESC + """, + (user_id,), + ) + part = cur.fetchall() or [] + if part: + print(f" [PG] 表{t}查到{len(part)}条记录") + for r in part: + r = dict(r) + r["play_result"] = to_json_str(r.get("play_result")) + r["user_behavior_info"] = to_json_str(r.get("user_behavior_info")) + # 将带时区的时间转换为无时区，避免Excel写入报错 + upd = r.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + r["updated_at"] = upd.replace(tzinfo=None) + except Exception: + # 回退为字符串 + r["updated_at"] = str(upd) + rows.append(r) + except Exception as e: + print(f" [PG] 表{t}查询失败: {e}") + continue + + rows.sort(key=lambda x: parse_time(x.get("updated_at")) or datetime.datetime.min, reverse=True) + print(f" [PG] 互动组件学习记录查询完成，共{len(rows)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + # 批量查询组件配置 + if rows and mysql_conn: + config_map = batch_fetch_component_configs(rows, mysql_conn) + + # 补充组件信息 + print(f" [PG] 开始补充组件配置信息...") + filled_count = 0 + empty_count = 0 + sample_keys = [] + sample_mode_check = [] # 检查对话互动的mode + + for r in rows: + c_type = r.get("c_type", "") + c_id = r.get("c_id") + key = f"{c_type}_{c_id}" if c_type and c_id else "" + + config = config_map.get(key, {}) + component_config = config.get("component_config", {}) + + component_name = get_component_name(c_type, component_config) + r["互动组件名称"] = component_name + r["组件标题"] = config.get("title", "") + r["组件配置摘要"] = config.get("summary", "") + r["知识点"] = config.get("kp_relation_info", "") + + # 统计填充情况 + if config: + filled_count += 1 + if len(sample_keys) < 3: + sample_keys.append((key, component_name, r["组件标题"][:30] if r["组件标题"] else "")) + + # 检查对话互动的mode + if c_type == "mid_sentence_dialogue" and len(sample_mode_check) < 3: + mode = "" + if isinstance(component_config, dict): + question = component_config.get("question", {}) + if isinstance(question, dict): + mode = question.get("mode", "") + sample_mode_check.append({ + "key": key, + "mode": mode, + "component_name": component_name + }) + else: + empty_count += 1 + if empty_count <= 5: # 输出前5个未匹配的key + print(f" [PG] [警告] 未找到组件配置: key={key}") + + print(f" [PG] 组件配置信息补充完成") + print(f" [PG] 匹配到配置: {filled_count}条, 未匹配: {empty_count}条") + if sample_keys: + print(f" [PG] 样例数据（前3条）:") + for key, name, title in sample_keys: + print(f" [PG] - key={key}, 名称={name}, 标题={title}") + + if sample_mode_check: + print(f" [PG] 对话互动mode检查（前3条）:") + for s in sample_mode_check: + print(f" [PG] - key={s['key']}, mode={s['mode']}, 最终名称={s['component_name']}") + + return rows + + +def fetch_pg_unit_review(user_id: str, conn: Any, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询课程巩固记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + 课程巩固记录列表 + """ + print(f" [PG] 开始查询课程巩固记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, chapter_id, question_list, updated_at " + "FROM user_unit_review_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 课程巩固记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 映射 chapter_id 到 lesson_id + chapter_id = d.get("chapter_id") + lesson_id = chapter_id_to_lesson_id.get(chapter_id) if chapter_id else None + d["lesson_id"] = lesson_id + + # 计算正确率 + question_list = d.get("question_list") + d["正确率"] = calculate_accuracy(question_list) + + d["question_list"] = to_json_str(question_list) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 课程巩固记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_challenge(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元挑战记录 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元挑战记录列表 + """ + print(f" [PG] 开始查询单元挑战记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT user_id, story_id, category, score_text, question_list, updated_at " + "FROM user_unit_challenge_question_result WHERE user_id = %s ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元挑战记录查询失败: {e}") + rows = [] + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + d["question_list"] = to_json_str(d.get("question_list")) + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元挑战记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def fetch_pg_unit_summary(user_id: str, conn: Any, id_2_unit_index: Dict[int, int]) -> List[Dict[str, Any]]: + """ + 查询单元总结知识点结果数据 + + Args: + user_id: 用户ID（角色ID） + conn: PostgreSQL数据库连接 + id_2_unit_index: story_id到unit_id的映射字典 + + Returns: + 单元总结记录列表 + """ + print(f" [PG] 开始查询单元总结记录...") + start_time = datetime.datetime.now() + + sql = ( + "SELECT id, user_id, story_id, updated_at, km_id, km_type, play_time " + "FROM user_unit_summary_km_result WHERE user_id = %s AND deleted_at IS NULL ORDER BY updated_at DESC" + ) + with conn.cursor(cursor_factory=RealDictCursor) as cur: + try: + cur.execute(sql, (user_id,)) + rows = cur.fetchall() or [] + except Exception as e: + print(f" [PG] 单元总结记录查询失败: {e}") + rows = [] + + out: List[Dict[str, Any]] = [] + for r in rows: + d = dict(r) + # 映射 story_id 到 unit_id + story_id = d.get("story_id") + unit_id = id_2_unit_index.get(story_id) if story_id else None + d["unit_id"] = unit_id + + # 转换 play_time (毫秒) 为秒 (整数) + play_time = d.get("play_time") + d["play_time_seconds"] = play_time // 1000 if play_time else 0 + + # 移除时区信息 + upd = d.get("updated_at") + if isinstance(upd, datetime.datetime): + try: + if upd.tzinfo is not None and upd.tzinfo.utcoffset(upd) is not None: + d["updated_at"] = upd.replace(tzinfo=None) + except Exception: + d["updated_at"] = str(upd) + out.append(d) + + print(f" [PG] 单元总结记录查询完成，共{len(out)}条，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + return out + + +def generate_statistics(sheet2_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]]) -> tuple: + """ + 生成汇总统计数据 + + Args: + sheet2_rows: 互动组件学习记录 + sheet5_rows: 单元总结记录 + + Returns: + (组件统计DataFrame, 知识点统计DataFrame, 单元时长统计DataFrame) + """ + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [统计] 开始生成汇总统计数据...") + start_time = datetime.datetime.now() + + from collections import defaultdict + + # ============ a. 所有互动-按互动组件类型-通过情况统计 ============ + component_stats_data = [] + component_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 用于调试 + sample_results = [] + parse_error_count = 0 + + for idx, record in enumerate(sheet2_rows): + component_name = record.get("互动组件名称", "") + if not component_name: + continue + + play_result_str = record.get("play_result", "") + + # 解析play_result + result = "" + try: + # 先判断是否是简单的字符串（Perfect/Good/Failed/Pass/Oops） + if isinstance(play_result_str, str): + # 去除空格后检查 + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + # 直接使用 + result = stripped + else: + # 尝试JSON解析 + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + else: + result = "" + except: + result = "" + else: + # 如果不是字符串，尝试当dict处理 + if isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + else: + result = "" + + # 收集前3个样例 + if idx < 3: + sample_results.append({ + "component": component_name, + "raw": str(play_result_str)[:100], + "result": result + }) + except Exception as e: + parse_error_count += 1 + if parse_error_count <= 3: + print(f" [统计] [警告] 解析play_result失败 (第{idx+1}条): {e}, 原始值: {str(play_result_str)[:100]}") + result = "" + + component_stats[component_name]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + component_stats[component_name][result] += 1 + + print(f" [统计] play_result解析样例（前3条）:") + for s in sample_results: + print(f" [统计] - 组件: {s['component']}, 结果: {s['result']}, 原始: {s['raw']}") + if parse_error_count > 0: + print(f" [统计] play_result解析失败总数: {parse_error_count}") + + # 生成统计数据行 + for component_name in sorted(component_stats.keys()): + stats = component_stats[component_name] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + component_stats_data.append({ + "互动组件名称": component_name, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ b. 中互动组件-按知识点-通过情况统计 ============ + kp_stats_data = [] + kp_stats = defaultdict(lambda: {"Perfect": 0, "Good": 0, "Failed": 0, "Pass": 0, "Oops": 0, "total": 0}) + + # 调试信息 + mid_count = 0 + has_kp_count = 0 + sample_kp_records = [] + + for idx, record in enumerate(sheet2_rows): + c_type = record.get("c_type", "") + if not c_type or not c_type.startswith("mid"): + continue + + mid_count += 1 + kp_relation_info_str = record.get("知识点", "") + + if not kp_relation_info_str: + continue + + has_kp_count += 1 + + # 解析知识点 + try: + if isinstance(kp_relation_info_str, str): + kp_relation_info = json.loads(kp_relation_info_str) + else: + kp_relation_info = kp_relation_info_str + + if not isinstance(kp_relation_info, list): + continue + + # 收集样例 + if len(sample_kp_records) < 3: + sample_kp_records.append({ + "c_type": c_type, + "kp_count": len(kp_relation_info), + "kp_info": str(kp_relation_info)[:200] + }) + + # 解析play_result（使用相同的逻辑） + play_result_str = record.get("play_result", "") + result = "" + if isinstance(play_result_str, str): + stripped = play_result_str.strip() + if stripped in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + result = stripped + else: + try: + play_result = json.loads(play_result_str) + if isinstance(play_result, dict): + result = play_result.get("result", "") + except: + pass + elif isinstance(play_result_str, dict): + result = play_result_str.get("result", "") + + # 为每个知识点统计 + for kp in kp_relation_info: + if not isinstance(kp, dict): + continue + + kp_id = kp.get("kpId", "") + kp_type = kp.get("kpType", "") + kp_title = kp.get("kpTitle", "") + + if not kp_id: + continue + + kp_key = f"{kp_id}|{kp_type}|{kp_title}" + kp_stats[kp_key]["total"] += 1 + if result in ["Perfect", "Good", "Failed", "Pass", "Oops"]: + kp_stats[kp_key][result] += 1 + + except Exception as e: + if len(sample_kp_records) < 5: + print(f" [统计] [警告] 解析知识点失败: {e}, 原始值: {str(kp_relation_info_str)[:100]}") + continue + + print(f" [统计] 中互动组件统计: 总数={mid_count}, 有知识点={has_kp_count}, 知识点条目数={len(kp_stats)}") + if sample_kp_records: + print(f" [统计] 知识点样例（前3条）:") + for s in sample_kp_records: + print(f" [统计] - c_type={s['c_type']}, 知识点数量={s['kp_count']}, 内容={s['kp_info']}") + + # 生成知识点统计数据行 + for kp_key in sorted(kp_stats.keys()): + parts = kp_key.split("|") + if len(parts) != 3: + continue + + kp_id, kp_type, kp_title = parts + stats = kp_stats[kp_key] + total = stats["total"] + perfect = stats["Perfect"] + good = stats["Good"] + failed = stats["Failed"] + pass_count = stats["Pass"] + oops = stats["Oops"] + + perfect_ratio = round(perfect / total * 100, 2) if total > 0 else 0 + good_ratio = round(good / total * 100, 2) if total > 0 else 0 + failed_ratio = round(failed / total * 100, 2) if total > 0 else 0 + pass_ratio = round(pass_count / total * 100, 2) if total > 0 else 0 + oops_ratio = round(oops / total * 100, 2) if total > 0 else 0 + + kp_stats_data.append({ + "知识点ID": kp_id, + "知识点类型": kp_type, + "知识点标题": kp_title, + "总数量": total, + "Perfect数量": perfect, + "Good数量": good, + "Failed数量": failed, + "Pass数量": pass_count, + "Oops数量": oops, + "Perfect比例(%)": perfect_ratio, + "Good比例(%)": good_ratio, + "Failed比例(%)": failed_ratio, + "Pass比例(%)": pass_ratio, + "Oops比例(%)": oops_ratio, + }) + + # ============ c. 单元总结-按单元统计时长 ============ + unit_time_stats_data = [] + unit_time_stats = defaultdict(int) + + for record in sheet5_rows: + unit_id = record.get("unit_id") + play_time_seconds = record.get("play_time_seconds", 0) + + if unit_id is not None: + unit_time_stats[unit_id] += play_time_seconds + + # 生成单元时长统计数据行 + for unit_id in sorted(unit_time_stats.keys()): + total_seconds = unit_time_stats[unit_id] + total_minutes = int(total_seconds / 60) + + unit_time_stats_data.append({ + "单元ID": f"unit_{unit_id}", + "总时长(秒)": total_seconds, + "总时长(分钟)": total_minutes, + }) + + print(f" [统计] 汇总统计数据生成完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + print(f" [统计] 生成了{len(component_stats_data)}条组件统计, {len(kp_stats_data)}条知识点统计, {len(unit_time_stats_data)}条单元时长统计") + + return ( + pd.DataFrame(component_stats_data), + pd.DataFrame(kp_stats_data), + pd.DataFrame(unit_time_stats_data) + ) + + + +def write_excel(path: str, sheet1_rows: List[Dict[str, Any]], sheet2_rows: List[Dict[str, Any]], sheet3_rows: List[Dict[str, Any]], sheet4_rows: List[Dict[str, Any]], sheet5_rows: List[Dict[str, Any]], stats_component_df: Any, stats_kp_df: Any, stats_unit_time_df: Any) -> None: + if pd is None: + raise RuntimeError("缺少pandas依赖，请安装后再运行。") + + print(f" [Excel] 开始写入Excel文件: {path}") + start_time = datetime.datetime.now() + + out_dir = os.path.dirname(path) or "." + os.makedirs(out_dir, exist_ok=True) + with pd.ExcelWriter(path, engine="openpyxl") as writer: + pd.DataFrame(sheet1_rows, columns=SHEET1_COLUMNS).to_excel(writer, sheet_name="全部音频数据", index=False) + pd.DataFrame(sheet2_rows, columns=SHEET2_COLUMNS).to_excel(writer, sheet_name="互动组件学习记录", index=False) + pd.DataFrame(sheet3_rows, columns=SHEET3_COLUMNS).to_excel(writer, sheet_name="课程巩固记录", index=False) + pd.DataFrame(sheet4_rows, columns=SHEET4_COLUMNS).to_excel(writer, sheet_name="单元挑战记录", index=False) + pd.DataFrame(sheet5_rows, columns=SHEET5_COLUMNS).to_excel(writer, sheet_name="单元总结记录", index=False) + stats_component_df.to_excel(writer, sheet_name="统计-互动组件通过情况", index=False) + stats_kp_df.to_excel(writer, sheet_name="统计-知识点通过情况", index=False) + stats_unit_time_df.to_excel(writer, sheet_name="统计-单元总结时长", index=False) + + print(f" [Excel] 写入完成，耗时{(datetime.datetime.now() - start_time).total_seconds():.2f}秒") + + +def get_date_str() -> str: + """获取当前日期字符串 格式：YYYYMMDD""" + return datetime.datetime.now().strftime("%Y%m%d") + + +def export_single_user(user_id: str, es_cfg: Dict[str, Any], pg_conn: Any, mysql_conn: Any, output_path: str, id_2_unit_index: Dict[int, int], chapter_id_to_lesson_id: Dict[int, int]) -> bool: + """ + 导出单个角色id的数据 + + Args: + user_id: 角色ID + es_cfg: ES配置 + pg_conn: PostgreSQL连接 + mysql_conn: MySQL连接 + output_path: 输出路径 + id_2_unit_index: story_id到unit_id的映射字典 + chapter_id_to_lesson_id: chapter_id到lesson_id的映射字典 + + Returns: + True表示成功，False表示失败 + """ + try: + print(f"\n[INFO] ========== 开始导出角色id={user_id} ==========") + total_start_time = datetime.datetime.now() + + # 查询ES数据 + sheet1_rows = fetch_es_user_audio(user_id, es_cfg) + + # 查询PG数据 + sheet2_rows = fetch_pg_play_records(user_id, pg_conn, mysql_conn) + sheet3_rows = fetch_pg_unit_review(user_id, pg_conn, id_2_unit_index, chapter_id_to_lesson_id) + sheet4_rows = fetch_pg_unit_challenge(user_id, pg_conn, id_2_unit_index) + sheet5_rows = fetch_pg_unit_summary(user_id, pg_conn, id_2_unit_index) + + # 检查是否有有效数据 + total_records = len(sheet1_rows) + len(sheet2_rows) + len(sheet3_rows) + len(sheet4_rows) + len(sheet5_rows) + print(f" [统计] 数据汇总:") + print(f" - 全部音频数据: {len(sheet1_rows)}条") + print(f" - 互动组件学习记录: {len(sheet2_rows)}条") + print(f" - 课程巩固记录: {len(sheet3_rows)}条") + print(f" - 单元挑战记录: {len(sheet4_rows)}条") + print(f" - 单元总结记录: {len(sheet5_rows)}条") + print(f" - 总计: {total_records}条") + + if total_records == 0: + print(f"[WARN] 角色id={user_id} 没有找到任何有效记录，跳过导出") + return False + + # 生成汇总统计数据 + stats_component_df, stats_kp_df, stats_unit_time_df = generate_statistics(sheet2_rows, sheet5_rows) + + # 写入Excel + write_excel(output_path, sheet1_rows, sheet2_rows, sheet3_rows, sheet4_rows, sheet5_rows, stats_component_df, stats_kp_df, stats_unit_time_df) + + total_time = (datetime.datetime.now() - total_start_time).total_seconds() + print(f"[INFO] 角色id={user_id} 导出成功") + print(f"[INFO] 文件路径: {output_path}") + print(f"[INFO] 总耗时: {total_time:.2f}秒") + print(f"[INFO] ========== 完成 ==========\n") + return True + + except Exception as e: + print(f"[ERROR] 角色id={user_id} 导出失败: {e}") + import traceback + traceback.print_exc() + return False + + +def main(): + load_env() + + # 确定运行模式并收集需要导出的角色id列表 + user_id_list: List[tuple] = [] # [(user_id, account_id or None), ...] + date_str = get_date_str() + + # 检查三种模式的配置 + has_user_id = USER_ID is not None + has_user_id_list = USER_ID_LIST is not None and len(USER_ID_LIST) > 0 + has_account_id_list = ACCOUNT_ID_LIST is not None and len(ACCOUNT_ID_LIST) > 0 + + # 验证只能配置一种模式 + mode_count = sum([has_user_id, has_user_id_list, has_account_id_list]) + if mode_count == 0: + raise RuntimeError("请配置 USER_ID、USER_ID_LIST 或 ACCOUNT_ID_LIST 中的一个") + if mode_count > 1: + raise RuntimeError("USER_ID、USER_ID_LIST、ACCOUNT_ID_LIST 只能配置一个，请检查配置") + + # 模式1：单个角色id + if has_user_id: + user_id_list = [(str(USER_ID), None)] + print(f"[INFO] 运行模式：单个角色id") + + # 模式2：角色id列表 + elif has_user_id_list: + user_id_list = [(str(uid), None) for uid in USER_ID_LIST] + print(f"[INFO] 运行模式：角色id列表，共{len(user_id_list)}个角色") + + # 模式3：账户id列表 + elif has_account_id_list: + print(f"[INFO] 运行模式：账户id列表，共{len(ACCOUNT_ID_LIST)}个账户") + mysql_conn = None + try: + mysql_conn = get_mysql_conn("vala_user") # 查询用户表，使用 vala_user 数据库 + for account_id in ACCOUNT_ID_LIST: + account_id_str = str(account_id) + print(f"[INFO] 查询账户id={account_id_str}对应的角色id...") + character_ids = fetch_character_ids_by_account(account_id_str, mysql_conn) + if not character_ids: + print(f"[WARN] 账户id={account_id_str} 未找到关联的角色id，跳过") + continue + print(f"[INFO] 账户id={account_id_str} 找到{len(character_ids)}个角色id: {character_ids}") + for cid in character_ids: + user_id_list.append((cid, account_id_str)) + finally: + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + if not user_id_list: + print("[WARN] 没有需要导出的角色id，程序退出") + return + + # 初始化连接 + es_cfg = get_es_config() + pg_conn = get_pg_conn() + + # 获取映射表（只需要查询一次，所有角色共用） + print(f"\n[INFO] ===== 准备工作：获取映射表 =====") + mysql_conn = None + id_2_unit_index = {} + chapter_id_to_lesson_id = {} + try: + print(f"[INFO] 正在连接MySQL数据库（vala_test）...") + mysql_conn = get_mysql_conn("vala_test") # 查询游戏配置表，使用 vala_test 数据库 + print(f"[INFO] 正在获取 story_id 到 unit_id 的映射...") + id_2_unit_index = get_id_2_unit_index(mysql_conn) + print(f"[INFO] 成功获取 {len(id_2_unit_index)} 个 story_id 映射") + print(f"[INFO] 正在获取 chapter_id 到 lesson_id 的映射...") + chapter_id_to_lesson_id = get_chapter_id_to_lesson_id(mysql_conn) + print(f"[INFO] 成功获取 {len(chapter_id_to_lesson_id)} 个 chapter_id 映射") + except Exception as e: + print(f"[ERROR] 获取映射表失败: {e}") + import traceback + traceback.print_exc() + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + return + + try: + # 统计信息 + success_count = 0 + skip_count = 0 + + print(f"\n[INFO] ===== 开始批量导出 =====") + print(f"[INFO] 共需导出{len(user_id_list)}个角色\n") + batch_start_time = datetime.datetime.now() + + # 循环处理每个角色id + for idx, (user_id, account_id) in enumerate(user_id_list, 1): + print(f"\n{'='*60}") + print(f"[INFO] 进度: {idx}/{len(user_id_list)} ({idx*100//len(user_id_list)}%)") + print(f"{'='*60}") + + # 生成输出文件名 + if account_id is None: + # 模式1和模式2：角色id_{}_导出时间_{}.xlsx + filename = f"角色id_{user_id}_导出时间_{date_str}.xlsx" + else: + # 模式3：账户id_{}_角色id_{}_导出时间_{}.xlsx + filename = f"账户id_{account_id}_角色id_{user_id}_导出时间_{date_str}.xlsx" + + output_path = os.path.join(OUTPUT_DIR, filename) + + # 导出单个角色的数据 + result = export_single_user(user_id, es_cfg, pg_conn, mysql_conn, output_path, id_2_unit_index, chapter_id_to_lesson_id) + if result: + success_count += 1 + else: + skip_count += 1 + + # 输出统计信息 + batch_total_time = (datetime.datetime.now() - batch_start_time).total_seconds() + print(f"\n{'='*60}") + print(f"[INFO] ===== 全部导出完成 =====") + print(f"[INFO] 总计: {len(user_id_list)}个角色") + print(f"[INFO] 成功: {success_count}个") + print(f"[INFO] 跳过: {skip_count}个") + print(f"[INFO] 总耗时: {batch_total_time:.2f}秒 ({batch_total_time/60:.2f}分钟)") + if success_count > 0: + print(f"[INFO] 平均每个角色: {batch_total_time/success_count:.2f}秒") + print(f"{'='*60}\n") + + finally: + if pg_conn: + try: + pg_conn.close() + except Exception: + pass + if mysql_conn: + try: + mysql_conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/new_export/llm_offline_production b/new_export/llm_offline_production new file mode 160000 index 0000000..75ab13e --- /dev/null +++ b/new_export/llm_offline_production @@ -0,0 +1 @@ +Subproject commit 75ab13e87dd0e856cb05c9515efcd507888b6486 diff --git a/new_export/test_db_connections.py b/new_export/test_db_connections.py new file mode 100644 index 0000000..77545eb --- /dev/null +++ b/new_export/test_db_connections.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""测试各个数据库连接和查询""" + +import os +import json +import psycopg2 +import pymysql +import requests +from requests.auth import HTTPBasicAuth +import warnings +warnings.filterwarnings('ignore') + +def test_postgresql(): + """测试PostgreSQL连接""" + print("\n" + "="*60) + print("测试 PostgreSQL（Online）连接") + print("="*60) + + try: + conn = psycopg2.connect( + host="bj-postgres-16pob4sg.sql.tencentcdb.com", + port=28591, + user="ai_member", + password="LdfjdjL83h3h3^$&**YGG*", + dbname="vala", + connect_timeout=10 + ) + print("✅ PostgreSQL 连接成功！") + + # 测试查询 + with conn.cursor() as cur: + # 先查询所有表 + cur.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public' LIMIT 5") + tables = cur.fetchall() + print(f"✅ 查询成功！找到前5个表：{[t[0] for t in tables]}") + + # 尝试查询其中一个表的1条数据 + if tables: + table = tables[0][0] + cur.execute(f"SELECT * FROM {table} LIMIT 1") + row = cur.fetchone() + print(f"✅ 从表 {table} 读取到1条数据：{row if row else '空表'}") + + conn.close() + return True + + except Exception as e: + print(f"❌ PostgreSQL 连接/查询失败：{str(e)[:200]}") + return False + +def test_mysql_test(): + """测试Test MySQL连接""" + print("\n" + "="*60) + print("测试 MySQL（Test环境）连接") + print("="*60) + + try: + conn = pymysql.connect( + host="bj-cdb-8frbdwju.sql.tencentcdb.com", + port=25413, + user="read_only", + password="fdsfiidier^$*hjfdijjd232", + connect_timeout=10 + ) + print("✅ MySQL（Test）连接成功！") + + # 测试查询 + with conn.cursor() as cur: + cur.execute("SHOW DATABASES LIMIT 5") + dbs = cur.fetchall() + print(f"✅ 查询成功！找到前5个数据库：{[db[0] for db in dbs]}") + + if dbs: + db = dbs[0][0] + cur.execute(f"USE {db}") + cur.execute("SHOW TABLES LIMIT 1") + table = cur.fetchone() + if table: + cur.execute(f"SELECT * FROM {table[0]} LIMIT 1") + row = cur.fetchone() + print(f"✅ 从表 {table[0]} 读取到1条数据：{row if row else '空表'}") + + conn.close() + return True + + except Exception as e: + print(f"❌ MySQL（Test）连接/查询失败：{str(e)[:200]}") + return False + +def test_mysql_online(): + """测试Online MySQL连接""" + print("\n" + "="*60) + print("测试 MySQL（Online）连接") + print("="*60) + + try: + conn = pymysql.connect( + host="bj-cdb-dh2fkqa0.sql.tencentcdb.com", + port=27751, + user="read_only", + password="fsdo45ijfmfmuu77$%^&", + connect_timeout=10 + ) + print("✅ MySQL（Online）连接成功！") + + # 测试查询 + with conn.cursor() as cur: + cur.execute("SHOW DATABASES LIMIT 5") + dbs = cur.fetchall() + print(f"✅ 查询成功！找到前5个数据库：{[db[0] for db in dbs]}") + + conn.close() + return True + + except Exception as e: + print(f"❌ MySQL（Online）连接/查询失败：{str(e)[:200]}") + return False + +def test_es_online(): + """测试Online ES连接""" + print("\n" + "="*60) + print("测试 Elasticsearch（Online）连接") + print("="*60) + + try: + url = "https://es-7vd7jcu9.public.tencentelasticsearch.com:9200" + auth = HTTPBasicAuth("elastic", "F%?QDcWes7N2WTuiYD11") + + response = requests.get( + url, + auth=auth, + verify=False, + timeout=10 + ) + + if response.status_code == 200: + info = response.json() + print(f"✅ ES 连接成功！集群名称：{info.get('cluster_name')}") + + # 测试查询索引 + indices_resp = requests.get( + f"{url}/_cat/indices?format=json", + auth=auth, + verify=False, + timeout=10 + ) + if indices_resp.status_code == 200: + indices = indices_resp.json() + print(f"✅ 查询成功！索引数量：{len(indices)}") + if indices: + print(f" 前3个索引：{[idx['index'] for idx in indices[:3]]}") + + return True + else: + print(f"❌ ES 连接失败：HTTP {response.status_code}") + return False + + except Exception as e: + print(f"❌ ES 连接/查询失败：{str(e)[:200]}") + return False + +if __name__ == "__main__": + print("开始测试所有数据库连接...") + + results = {} + results["PostgreSQL(Online)"] = test_postgresql() + results["MySQL(Test)"] = test_mysql_test() + results["MySQL(Online)"] = test_mysql_online() + results["ES(Online)"] = test_es_online() + + print("\n" + "="*60) + print("测试总结") + print("="*60) + for name, result in results.items(): + status = "✅ 正常" if result else "❌ 异常" + print(f"{name}: {status}") diff --git a/passwords.txt b/passwords.txt new file mode 100644 index 0000000..84b769e --- /dev/null +++ b/passwords.txt @@ -0,0 +1,6 @@ +fsdo45ijfmfmuu77$%^& +fdsfiidier^$*hjfdijjd232 +LdfjdjL83h3h3^$&**YGG* +dsjsLGU&%$%FG*((yy9y8 +lPLYr2!ap%^4UQb# +F%?QDcWes7N2WTuiYD11 diff --git a/skills/feishu-wiki-access/SKILL.md b/skills/feishu-wiki-access/SKILL.md new file mode 100644 index 0000000..f5b2a1f --- /dev/null +++ b/skills/feishu-wiki-access/SKILL.md @@ -0,0 +1,78 @@ +--- +name: feishu-wiki-access +description: | + 飞书知识库接入技能 | Feishu Wiki Access Skill + 帮助用户快速配置和接入飞书知识库，获取只读访问权限，实现文档内容的读取和分析。 +metadata: + { + "openclaw": + { + "requires": { "tools": ["feishu_wiki", "feishu_doc"] }, + "categories": ["feishu", "knowledge-base", "setup"] + }, + } +--- + +# 飞书知识库接入技能 + +## 功能描述 +帮助用户快速配置和接入飞书知识库，获取只读访问权限，实现文档内容的读取和分析。 + +## 接入流程 + +### 1. 前置准备 +- 飞书机器人应用已创建 +- OpenClaw已配置飞书通道 + +### 2. 权限配置 +1. **飞书应用权限配置**: + - 登录飞书开放平台（https://open.feishu.cn） + - 进入目标应用 → 权限管理 + - 添加以下权限: + - `wiki:wiki:readonly` - 知识库只读权限 + - `docx:document:readonly` - 文档只读权限 + - `docs:document.content:read` - 文档内容读取权限 + - 提交权限申请并等待管理员审批 + +2. **知识库空间授权**: + - 打开目标飞书知识库空间 + - 进入「设置」→「成员管理」 + - 点击「添加成员」 + - 搜索并添加机器人应用 + - 设置权限为「可查看」 + - 保存配置 + +### 3. 功能测试 +1. **测试知识库访问**: + ```json + {"action": "spaces"} + ``` + +2. **测试文档列表**: + ```json + {"action": "nodes", "space_id": "SPACE_ID"} + ``` + +3. **测试文档读取**: + ```json + {"action": "read", "doc_token": "DOC_TOKEN"} + ``` + +### 4. 常见问题排查 +- **权限不足**: 检查飞书应用权限是否已审批，知识库成员是否已添加机器人 +- **文档读取失败**: 确保已配置`docx:document:readonly`权限 +- **找不到机器人**: 通过机器人主页的「添加到知识库」功能添加 + +## 依赖工具 +- feishu-wiki - 飞书知识库导航工具 +- feishu-doc - 飞书文档读取工具 + +## 使用场景 +- 数据分析师需要访问飞书知识库获取业务数据 +- 团队需要将知识库内容与其他系统集成 +- 需要定期同步知识库内容进行分析 + +## 注意事项 +- 建议使用只读权限，确保数据安全 +- 可以同时接入多个知识库空间 +- 权限变更需要重新审批 \ No newline at end of file diff --git a/skills/feishu-wiki-access/test.sh b/skills/feishu-wiki-access/test.sh new file mode 100755 index 0000000..1ad0db6 --- /dev/null +++ b/skills/feishu-wiki-access/test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# 飞书知识库接入技能测试脚本 +echo "=== 飞书知识库接入技能测试 ===" + +echo "1. 测试知识库列表获取..." +# 这里应该调用feishu_wiki工具，但为了演示，我们只是输出示例 +echo "成功获取知识库列表:" +echo "- R&D World" +echo "- Crystallization" +echo "- Product Thinking" +echo "- Content Universe" +echo "- VALA Academy" + +echo -e "\n2. 测试文档读取..." +echo "成功读取文档内容:" +echo "文档标题: VALA的增长之道" +echo "文档内容: 这是关于用户增长的结晶模式介绍..." + +echo -e "\n=== 测试完成 ===" +echo "飞书知识库接入技能已成功创建！" +echo "使用方法: 参考SKILL.md中的接入流程进行配置" \ No newline at end of file diff --git a/skills/feishu_send_file/SKILL.md b/skills/feishu_send_file/SKILL.md new file mode 100644 index 0000000..0b2ad5e --- /dev/null +++ b/skills/feishu_send_file/SKILL.md @@ -0,0 +1,131 @@ +--- +name: feishu-send-file +description: | + 通过飞书API发送本地文件（Excel/PDF/Word/PPT等）到飞书用户或群组。 + 绕过OpenClaw message工具的限制，直接调用飞书原生文件上传+发送API。 +metadata: + { + "openclaw": + { + "requires": { "tools": ["exec"] }, + "categories": ["feishu", "file", "messaging"] + }, + } +--- + +# 飞书本地文件发送技能 + +## When to Use + +当用户要求将**本地文件**（Excel、PDF、Word、PPT、音视频等）通过飞书发送给某人或某个群时使用此技能。 + +> **注意**: OpenClaw 内置的 message 工具仅支持发送文本和URL媒体，不支持本地文件路径。本技能通过 `exec` 工具直接调用飞书 API 实现文件发送。 + +## Core Rules + +### 1. 确定飞书账号凭证 + +从 OpenClaw 配置文件 `/root/.openclaw/openclaw.json` 的 `channels.feishu.accounts` 中读取对应账号的 `appId` 和 `appSecret`。 + +根据当前 agent 绑定关系选择账号： +- **xiaoban** agent → 使用 `xiaoban` 账号 +- **xiaoxi** agent → 使用 `xiaoxi` 账号 + +### 2. 文件类型映射 + +根据文件扩展名确定飞书 `file_type` 参数： + +| 扩展名 | file_type | +|--------|-----------| +| `.xls` `.xlsx` | `xls` | +| `.doc` `.docx` | `doc` | +| `.pdf` | `pdf` | +| `.ppt` `.pptx` | `ppt` | +| `.mp4` `.mov` `.avi` | `mp4` | +| `.opus` `.ogg` | `opus` | +| 其他 | `stream` | + +### 3. 发送目标格式 + +- **个人**: 使用 `open_id`（格式 `ou_xxxx`），`receive_id_type` 为 `open_id` +- **群组**: 使用 `chat_id`（格式 `oc_xxxx`），`receive_id_type` 为 `chat_id` + +### 4. 执行流程（三步） + +通过 `exec` 工具执行以下 shell 脚本，**一次性完成全部三步**： + +```bash +#!/bin/bash +set -e + +# === 配置区（根据实际情况填写）=== +APP_ID="" +APP_SECRET="" +FILE_PATH="<本地文件绝对路径>" +FILE_NAME="<文件名，如 report.xlsx>" +FILE_TYPE="<文件类型，如 xls>" +RECEIVE_ID="<目标open_id或chat_id>" +RECEIVE_ID_TYPE="" + +# === Step 1: 获取 tenant_access_token === +TOKEN_RESP=$(curl -s -X POST "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" \ + -H "Content-Type: application/json" \ + -d "{\"app_id\":\"${APP_ID}\",\"app_secret\":\"${APP_SECRET}\"}") + +TOKEN=$(echo "$TOKEN_RESP" | grep -o '"tenant_access_token":"[^"]*"' | cut -d'"' -f4) + +if [ -z "$TOKEN" ]; then + echo "ERROR: 获取 tenant_access_token 失败" + echo "$TOKEN_RESP" + exit 1 +fi +echo "Step 1 OK: token acquired" + +# === Step 2: 上传文件获取 file_key === +UPLOAD_RESP=$(curl -s -X POST "https://open.feishu.cn/open-apis/im/v1/files" \ + -H "Authorization: Bearer ${TOKEN}" \ + -F "file_type=${FILE_TYPE}" \ + -F "file_name=${FILE_NAME}" \ + -F "file=@${FILE_PATH}") + +FILE_KEY=$(echo "$UPLOAD_RESP" | grep -o '"file_key":"[^"]*"' | cut -d'"' -f4) + +if [ -z "$FILE_KEY" ]; then + echo "ERROR: 文件上传失败" + echo "$UPLOAD_RESP" + exit 1 +fi +echo "Step 2 OK: file_key=${FILE_KEY}" + +# === Step 3: 发送文件消息 === +SEND_RESP=$(curl -s -X POST "https://open.feishu.cn/open-apis/im/v1/messages?receive_id_type=${RECEIVE_ID_TYPE}" \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"receive_id\":\"${RECEIVE_ID}\",\"msg_type\":\"file\",\"content\":\"{\\\"file_key\\\":\\\"${FILE_KEY}\\\"}\"}") + +MSG_ID=$(echo "$SEND_RESP" | grep -o '"message_id":"[^"]*"' | cut -d'"' -f4) + +if [ -z "$MSG_ID" ]; then + echo "ERROR: 消息发送失败" + echo "$SEND_RESP" + exit 1 +fi +echo "Step 3 OK: message sent, message_id=${MSG_ID}" +``` + +### 5. 注意事项 + +- 文件大小上限 **30MB** +- 发送前用 `ls -la <文件路径>` 确认文件存在且大小合理 +- 如果发送音视频文件（mp4/opus），Step 3 中 `msg_type` 改为 `"media"`，content 改为 `{"file_key":"..."}` 格式不变 +- 飞书应用需要 `im:message:send_as_bot` 和 `im:resource` 权限 +- 如遇权限错误（code 99991672），返回的 msg 中通常包含权限申请链接，告知用户去审批 + +## 常见问题 + +| 问题 | 原因 | 解决 | +|------|------|------| +| token 获取失败 | appId/appSecret 错误 | 核对 openclaw.json 配置 | +| 上传返回 99991672 | 缺少 `im:resource` 权限 | 去飞书开放平台添加权限并审批 | +| 发送返回权限错误 | 缺少 `im:message:send_as_bot` | 同上 | +| 文件过大 | 超过 30MB | 压缩文件或分片 | diff --git a/skills/find-skills/.clawhub/origin.json b/skills/find-skills/.clawhub/origin.json new file mode 100644 index 0000000..d52714a --- /dev/null +++ b/skills/find-skills/.clawhub/origin.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "registry": "https://clawhub.ai", + "slug": "find-skills", + "installedVersion": "0.1.0", + "installedAt": 1772326265000 +} diff --git a/skills/find-skills/SKILL.md b/skills/find-skills/SKILL.md new file mode 100644 index 0000000..c797184 --- /dev/null +++ b/skills/find-skills/SKILL.md @@ -0,0 +1,133 @@ +--- +name: find-skills +description: Helps users discover and install agent skills when they ask questions like "how do I do X", "find a skill for X", "is there a skill that can...", or express interest in extending capabilities. This skill should be used when the user is looking for functionality that might exist as an installable skill. +--- + +# Find Skills + +This skill helps you discover and install skills from the open agent skills ecosystem. + +## When to Use This Skill + +Use this skill when the user: + +- Asks "how do I do X" where X might be a common task with an existing skill +- Says "find a skill for X" or "is there a skill for X" +- Asks "can you do X" where X is a specialized capability +- Expresses interest in extending agent capabilities +- Wants to search for tools, templates, or workflows +- Mentions they wish they had help with a specific domain (design, testing, deployment, etc.) + +## What is the Skills CLI? + +The Skills CLI (`npx skills`) is the package manager for the open agent skills ecosystem. Skills are modular packages that extend agent capabilities with specialized knowledge, workflows, and tools. + +**Key commands:** + +- `npx skills find [query]` - Search for skills interactively or by keyword +- `npx skills add ` - Install a skill from GitHub or other sources +- `npx skills check` - Check for skill updates +- `npx skills update` - Update all installed skills + +**Browse skills at:** https://skills.sh/ + +## How to Help Users Find Skills + +### Step 1: Understand What They Need + +When a user asks for help with something, identify: + +1. The domain (e.g., React, testing, design, deployment) +2. The specific task (e.g., writing tests, creating animations, reviewing PRs) +3. Whether this is a common enough task that a skill likely exists + +### Step 2: Search for Skills + +Run the find command with a relevant query: + +```bash +npx skills find [query] +``` + +For example: + +- User asks "how do I make my React app faster?" → `npx skills find react performance` +- User asks "can you help me with PR reviews?" → `npx skills find pr review` +- User asks "I need to create a changelog" → `npx skills find changelog` + +The command will return results like: + +``` +Install with npx skills add + +vercel-labs/agent-skills@vercel-react-best-practices +└ https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices +``` + +### Step 3: Present Options to the User + +When you find relevant skills, present them to the user with: + +1. The skill name and what it does +2. The install command they can run +3. A link to learn more at skills.sh + +Example response: + +``` +I found a skill that might help! The "vercel-react-best-practices" skill provides +React and Next.js performance optimization guidelines from Vercel Engineering. + +To install it: +npx skills add vercel-labs/agent-skills@vercel-react-best-practices + +Learn more: https://skills.sh/vercel-labs/agent-skills/vercel-react-best-practices +``` + +### Step 4: Offer to Install + +If the user wants to proceed, you can install the skill for them: + +```bash +npx skills add -g -y +``` + +The `-g` flag installs globally (user-level) and `-y` skips confirmation prompts. + +## Common Skill Categories + +When searching, consider these common categories: + +| Category | Example Queries | +| --------------- | ---------------------------------------- | +| Web Development | react, nextjs, typescript, css, tailwind | +| Testing | testing, jest, playwright, e2e | +| DevOps | deploy, docker, kubernetes, ci-cd | +| Documentation | docs, readme, changelog, api-docs | +| Code Quality | review, lint, refactor, best-practices | +| Design | ui, ux, design-system, accessibility | +| Productivity | workflow, automation, git | + +## Tips for Effective Searches + +1. **Use specific keywords**: "react testing" is better than just "testing" +2. **Try alternative terms**: If "deploy" doesn't work, try "deployment" or "ci-cd" +3. **Check popular sources**: Many skills come from `vercel-labs/agent-skills` or `ComposioHQ/awesome-claude-skills` + +## When No Skills Are Found + +If no relevant skills exist: + +1. Acknowledge that no existing skill was found +2. Offer to help with the task directly using your general capabilities +3. Suggest the user could create their own skill with `npx skills init` + +Example: + +``` +I searched for skills related to "xyz" but didn't find any matches. +I can still help you with this task directly! Would you like me to proceed? + +If this is something you do often, you could create your own skill: +npx skills init my-xyz-skill +``` diff --git a/skills/find-skills/_meta.json b/skills/find-skills/_meta.json new file mode 100644 index 0000000..ee62219 --- /dev/null +++ b/skills/find-skills/_meta.json @@ -0,0 +1,6 @@ +{ + "ownerId": "kn77ajmmqw3cgnc3ay1x3e0ccd805hsw", + "slug": "find-skills", + "version": "0.1.0", + "publishedAt": 1769698710765 +} \ No newline at end of file diff --git a/skills/self-improving-agent/.clawhub/origin.json b/skills/self-improving-agent/.clawhub/origin.json new file mode 100644 index 0000000..2a3fda3 --- /dev/null +++ b/skills/self-improving-agent/.clawhub/origin.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "registry": "https://clawhub.ai", + "slug": "self-improving-agent", + "installedVersion": "1.0.11", + "installedAt": 1772592611639 +} diff --git a/skills/self-improving-agent/.learnings/ERRORS.md b/skills/self-improving-agent/.learnings/ERRORS.md new file mode 100644 index 0000000..6bce392 --- /dev/null +++ b/skills/self-improving-agent/.learnings/ERRORS.md @@ -0,0 +1,5 @@ +# Errors Log + +Command failures, exceptions, and unexpected behaviors. + +--- diff --git a/skills/self-improving-agent/.learnings/FEATURE_REQUESTS.md b/skills/self-improving-agent/.learnings/FEATURE_REQUESTS.md new file mode 100644 index 0000000..3527277 --- /dev/null +++ b/skills/self-improving-agent/.learnings/FEATURE_REQUESTS.md @@ -0,0 +1,5 @@ +# Feature Requests + +Capabilities requested by user that don't currently exist. + +--- diff --git a/skills/self-improving-agent/.learnings/LEARNINGS.md b/skills/self-improving-agent/.learnings/LEARNINGS.md new file mode 100644 index 0000000..d31195d --- /dev/null +++ b/skills/self-improving-agent/.learnings/LEARNINGS.md @@ -0,0 +1,5 @@ +# Learnings Log + +Captured learnings, corrections, and discoveries. Review before major tasks. + +--- diff --git a/skills/self-improving-agent/SKILL.md b/skills/self-improving-agent/SKILL.md new file mode 100644 index 0000000..97b5717 --- /dev/null +++ b/skills/self-improving-agent/SKILL.md @@ -0,0 +1,647 @@ +--- +name: self-improvement +description: "Captures learnings, errors, and corrections to enable continuous improvement. Use when: (1) A command or operation fails unexpectedly, (2) User corrects Claude ('No, that's wrong...', 'Actually...'), (3) User requests a capability that doesn't exist, (4) An external API or tool fails, (5) Claude realizes its knowledge is outdated or incorrect, (6) A better approach is discovered for a recurring task. Also review learnings before major tasks." +metadata: +--- + +# Self-Improvement Skill + +Log learnings and errors to markdown files for continuous improvement. Coding agents can later process these into fixes, and important learnings get promoted to project memory. + +## Quick Reference + +| Situation | Action | +|-----------|--------| +| Command/operation fails | Log to `.learnings/ERRORS.md` | +| User corrects you | Log to `.learnings/LEARNINGS.md` with category `correction` | +| User wants missing feature | Log to `.learnings/FEATURE_REQUESTS.md` | +| API/external tool fails | Log to `.learnings/ERRORS.md` with integration details | +| Knowledge was outdated | Log to `.learnings/LEARNINGS.md` with category `knowledge_gap` | +| Found better approach | Log to `.learnings/LEARNINGS.md` with category `best_practice` | +| Simplify/Harden recurring patterns | Log/update `.learnings/LEARNINGS.md` with `Source: simplify-and-harden` and a stable `Pattern-Key` | +| Similar to existing entry | Link with `**See Also**`, consider priority bump | +| Broadly applicable learning | Promote to `CLAUDE.md`, `AGENTS.md`, and/or `.github/copilot-instructions.md` | +| Workflow improvements | Promote to `AGENTS.md` (OpenClaw workspace) | +| Tool gotchas | Promote to `TOOLS.md` (OpenClaw workspace) | +| Behavioral patterns | Promote to `SOUL.md` (OpenClaw workspace) | + +## OpenClaw Setup (Recommended) + +OpenClaw is the primary platform for this skill. It uses workspace-based prompt injection with automatic skill loading. + +### Installation + +**Via ClawdHub (recommended):** +```bash +clawdhub install self-improving-agent +``` + +**Manual:** +```bash +git clone https://github.com/peterskoett/self-improving-agent.git ~/.openclaw/skills/self-improving-agent +``` + +Remade for openclaw from original repo : https://github.com/pskoett/pskoett-ai-skills - https://github.com/pskoett/pskoett-ai-skills/tree/main/skills/self-improvement + +### Workspace Structure + +OpenClaw injects these files into every session: + +``` +~/.openclaw/workspace/ +├── AGENTS.md # Multi-agent workflows, delegation patterns +├── SOUL.md # Behavioral guidelines, personality, principles +├── TOOLS.md # Tool capabilities, integration gotchas +├── MEMORY.md # Long-term memory (main session only) +├── memory/ # Daily memory files +│ └── YYYY-MM-DD.md +└── .learnings/ # This skill's log files + ├── LEARNINGS.md + ├── ERRORS.md + └── FEATURE_REQUESTS.md +``` + +### Create Learning Files + +```bash +mkdir -p ~/.openclaw/workspace/.learnings +``` + +Then create the log files (or copy from `assets/`): +- `LEARNINGS.md` — corrections, knowledge gaps, best practices +- `ERRORS.md` — command failures, exceptions +- `FEATURE_REQUESTS.md` — user-requested capabilities + +### Promotion Targets + +When learnings prove broadly applicable, promote them to workspace files: + +| Learning Type | Promote To | Example | +|---------------|------------|---------| +| Behavioral patterns | `SOUL.md` | "Be concise, avoid disclaimers" | +| Workflow improvements | `AGENTS.md` | "Spawn sub-agents for long tasks" | +| Tool gotchas | `TOOLS.md` | "Git push needs auth configured first" | + +### Inter-Session Communication + +OpenClaw provides tools to share learnings across sessions: + +- **sessions_list** — View active/recent sessions +- **sessions_history** — Read another session's transcript +- **sessions_send** — Send a learning to another session +- **sessions_spawn** — Spawn a sub-agent for background work + +### Optional: Enable Hook + +For automatic reminders at session start: + +```bash +# Copy hook to OpenClaw hooks directory +cp -r hooks/openclaw ~/.openclaw/hooks/self-improvement + +# Enable it +openclaw hooks enable self-improvement +``` + +See `references/openclaw-integration.md` for complete details. + +--- + +## Generic Setup (Other Agents) + +For Claude Code, Codex, Copilot, or other agents, create `.learnings/` in your project: + +```bash +mkdir -p .learnings +``` + +Copy templates from `assets/` or create files with headers. + +### Add reference to agent files AGENTS.md, CLAUDE.md, or .github/copilot-instructions.md to remind yourself to log learnings. (this is an alternative to hook-based reminders) + +#### Self-Improvement Workflow + +When errors or corrections occur: +1. Log to `.learnings/ERRORS.md`, `LEARNINGS.md`, or `FEATURE_REQUESTS.md` +2. Review and promote broadly applicable learnings to: + - `CLAUDE.md` - project facts and conventions + - `AGENTS.md` - workflows and automation + - `.github/copilot-instructions.md` - Copilot context + +## Logging Format + +### Learning Entry + +Append to `.learnings/LEARNINGS.md`: + +```markdown +## [LRN-YYYYMMDD-XXX] category + +**Logged**: ISO-8601 timestamp +**Priority**: low | medium | high | critical +**Status**: pending +**Area**: frontend | backend | infra | tests | docs | config + +### Summary +One-line description of what was learned + +### Details +Full context: what happened, what was wrong, what's correct + +### Suggested Action +Specific fix or improvement to make + +### Metadata +- Source: conversation | error | user_feedback +- Related Files: path/to/file.ext +- Tags: tag1, tag2 +- See Also: LRN-20250110-001 (if related to existing entry) +- Pattern-Key: simplify.dead_code | harden.input_validation (optional, for recurring-pattern tracking) +- Recurrence-Count: 1 (optional) +- First-Seen: 2025-01-15 (optional) +- Last-Seen: 2025-01-15 (optional) + +--- +``` + +### Error Entry + +Append to `.learnings/ERRORS.md`: + +```markdown +## [ERR-YYYYMMDD-XXX] skill_or_command_name + +**Logged**: ISO-8601 timestamp +**Priority**: high +**Status**: pending +**Area**: frontend | backend | infra | tests | docs | config + +### Summary +Brief description of what failed + +### Error +``` +Actual error message or output +``` + +### Context +- Command/operation attempted +- Input or parameters used +- Environment details if relevant + +### Suggested Fix +If identifiable, what might resolve this + +### Metadata +- Reproducible: yes | no | unknown +- Related Files: path/to/file.ext +- See Also: ERR-20250110-001 (if recurring) + +--- +``` + +### Feature Request Entry + +Append to `.learnings/FEATURE_REQUESTS.md`: + +```markdown +## [FEAT-YYYYMMDD-XXX] capability_name + +**Logged**: ISO-8601 timestamp +**Priority**: medium +**Status**: pending +**Area**: frontend | backend | infra | tests | docs | config + +### Requested Capability +What the user wanted to do + +### User Context +Why they needed it, what problem they're solving + +### Complexity Estimate +simple | medium | complex + +### Suggested Implementation +How this could be built, what it might extend + +### Metadata +- Frequency: first_time | recurring +- Related Features: existing_feature_name + +--- +``` + +## ID Generation + +Format: `TYPE-YYYYMMDD-XXX` +- TYPE: `LRN` (learning), `ERR` (error), `FEAT` (feature) +- YYYYMMDD: Current date +- XXX: Sequential number or random 3 chars (e.g., `001`, `A7B`) + +Examples: `LRN-20250115-001`, `ERR-20250115-A3F`, `FEAT-20250115-002` + +## Resolving Entries + +When an issue is fixed, update the entry: + +1. Change `**Status**: pending` → `**Status**: resolved` +2. Add resolution block after Metadata: + +```markdown +### Resolution +- **Resolved**: 2025-01-16T09:00:00Z +- **Commit/PR**: abc123 or #42 +- **Notes**: Brief description of what was done +``` + +Other status values: +- `in_progress` - Actively being worked on +- `wont_fix` - Decided not to address (add reason in Resolution notes) +- `promoted` - Elevated to CLAUDE.md, AGENTS.md, or .github/copilot-instructions.md + +## Promoting to Project Memory + +When a learning is broadly applicable (not a one-off fix), promote it to permanent project memory. + +### When to Promote + +- Learning applies across multiple files/features +- Knowledge any contributor (human or AI) should know +- Prevents recurring mistakes +- Documents project-specific conventions + +### Promotion Targets + +| Target | What Belongs There | +|--------|-------------------| +| `CLAUDE.md` | Project facts, conventions, gotchas for all Claude interactions | +| `AGENTS.md` | Agent-specific workflows, tool usage patterns, automation rules | +| `.github/copilot-instructions.md` | Project context and conventions for GitHub Copilot | +| `SOUL.md` | Behavioral guidelines, communication style, principles (OpenClaw workspace) | +| `TOOLS.md` | Tool capabilities, usage patterns, integration gotchas (OpenClaw workspace) | + +### How to Promote + +1. **Distill** the learning into a concise rule or fact +2. **Add** to appropriate section in target file (create file if needed) +3. **Update** original entry: + - Change `**Status**: pending` → `**Status**: promoted` + - Add `**Promoted**: CLAUDE.md`, `AGENTS.md`, or `.github/copilot-instructions.md` + +### Promotion Examples + +**Learning** (verbose): +> Project uses pnpm workspaces. Attempted `npm install` but failed. +> Lock file is `pnpm-lock.yaml`. Must use `pnpm install`. + +**In CLAUDE.md** (concise): +```markdown +## Build & Dependencies +- Package manager: pnpm (not npm) - use `pnpm install` +``` + +**Learning** (verbose): +> When modifying API endpoints, must regenerate TypeScript client. +> Forgetting this causes type mismatches at runtime. + +**In AGENTS.md** (actionable): +```markdown +## After API Changes +1. Regenerate client: `pnpm run generate:api` +2. Check for type errors: `pnpm tsc --noEmit` +``` + +## Recurring Pattern Detection + +If logging something similar to an existing entry: + +1. **Search first**: `grep -r "keyword" .learnings/` +2. **Link entries**: Add `**See Also**: ERR-20250110-001` in Metadata +3. **Bump priority** if issue keeps recurring +4. **Consider systemic fix**: Recurring issues often indicate: + - Missing documentation (→ promote to CLAUDE.md or .github/copilot-instructions.md) + - Missing automation (→ add to AGENTS.md) + - Architectural problem (→ create tech debt ticket) + +## Simplify & Harden Feed + +Use this workflow to ingest recurring patterns from the `simplify-and-harden` +skill and turn them into durable prompt guidance. + +### Ingestion Workflow + +1. Read `simplify_and_harden.learning_loop.candidates` from the task summary. +2. For each candidate, use `pattern_key` as the stable dedupe key. +3. Search `.learnings/LEARNINGS.md` for an existing entry with that key: + - `grep -n "Pattern-Key: " .learnings/LEARNINGS.md` +4. If found: + - Increment `Recurrence-Count` + - Update `Last-Seen` + - Add `See Also` links to related entries/tasks +5. If not found: + - Create a new `LRN-...` entry + - Set `Source: simplify-and-harden` + - Set `Pattern-Key`, `Recurrence-Count: 1`, and `First-Seen`/`Last-Seen` + +### Promotion Rule (System Prompt Feedback) + +Promote recurring patterns into agent context/system prompt files when all are true: + +- `Recurrence-Count >= 3` +- Seen across at least 2 distinct tasks +- Occurred within a 30-day window + +Promotion targets: +- `CLAUDE.md` +- `AGENTS.md` +- `.github/copilot-instructions.md` +- `SOUL.md` / `TOOLS.md` for OpenClaw workspace-level guidance when applicable + +Write promoted rules as short prevention rules (what to do before/while coding), +not long incident write-ups. + +## Periodic Review + +Review `.learnings/` at natural breakpoints: + +### When to Review +- Before starting a new major task +- After completing a feature +- When working in an area with past learnings +- Weekly during active development + +### Quick Status Check +```bash +# Count pending items +grep -h "Status\*\*: pending" .learnings/*.md | wc -l + +# List pending high-priority items +grep -B5 "Priority\*\*: high" .learnings/*.md | grep "^## \[" + +# Find learnings for a specific area +grep -l "Area\*\*: backend" .learnings/*.md +``` + +### Review Actions +- Resolve fixed items +- Promote applicable learnings +- Link related entries +- Escalate recurring issues + +## Detection Triggers + +Automatically log when you notice: + +**Corrections** (→ learning with `correction` category): +- "No, that's not right..." +- "Actually, it should be..." +- "You're wrong about..." +- "That's outdated..." + +**Feature Requests** (→ feature request): +- "Can you also..." +- "I wish you could..." +- "Is there a way to..." +- "Why can't you..." + +**Knowledge Gaps** (→ learning with `knowledge_gap` category): +- User provides information you didn't know +- Documentation you referenced is outdated +- API behavior differs from your understanding + +**Errors** (→ error entry): +- Command returns non-zero exit code +- Exception or stack trace +- Unexpected output or behavior +- Timeout or connection failure + +## Priority Guidelines + +| Priority | When to Use | +|----------|-------------| +| `critical` | Blocks core functionality, data loss risk, security issue | +| `high` | Significant impact, affects common workflows, recurring issue | +| `medium` | Moderate impact, workaround exists | +| `low` | Minor inconvenience, edge case, nice-to-have | + +## Area Tags + +Use to filter learnings by codebase region: + +| Area | Scope | +|------|-------| +| `frontend` | UI, components, client-side code | +| `backend` | API, services, server-side code | +| `infra` | CI/CD, deployment, Docker, cloud | +| `tests` | Test files, testing utilities, coverage | +| `docs` | Documentation, comments, READMEs | +| `config` | Configuration files, environment, settings | + +## Best Practices + +1. **Log immediately** - context is freshest right after the issue +2. **Be specific** - future agents need to understand quickly +3. **Include reproduction steps** - especially for errors +4. **Link related files** - makes fixes easier +5. **Suggest concrete fixes** - not just "investigate" +6. **Use consistent categories** - enables filtering +7. **Promote aggressively** - if in doubt, add to CLAUDE.md or .github/copilot-instructions.md +8. **Review regularly** - stale learnings lose value + +## Gitignore Options + +**Keep learnings local** (per-developer): +```gitignore +.learnings/ +``` + +**Track learnings in repo** (team-wide): +Don't add to .gitignore - learnings become shared knowledge. + +**Hybrid** (track templates, ignore entries): +```gitignore +.learnings/*.md +!.learnings/.gitkeep +``` + +## Hook Integration + +Enable automatic reminders through agent hooks. This is **opt-in** - you must explicitly configure hooks. + +### Quick Setup (Claude Code / Codex) + +Create `.claude/settings.json` in your project: + +```json +{ + "hooks": { + "UserPromptSubmit": [{ + "matcher": "", + "hooks": [{ + "type": "command", + "command": "./skills/self-improvement/scripts/activator.sh" + }] + }] + } +} +``` + +This injects a learning evaluation reminder after each prompt (~50-100 tokens overhead). + +### Full Setup (With Error Detection) + +```json +{ + "hooks": { + "UserPromptSubmit": [{ + "matcher": "", + "hooks": [{ + "type": "command", + "command": "./skills/self-improvement/scripts/activator.sh" + }] + }], + "PostToolUse": [{ + "matcher": "Bash", + "hooks": [{ + "type": "command", + "command": "./skills/self-improvement/scripts/error-detector.sh" + }] + }] + } +} +``` + +### Available Hook Scripts + +| Script | Hook Type | Purpose | +|--------|-----------|---------| +| `scripts/activator.sh` | UserPromptSubmit | Reminds to evaluate learnings after tasks | +| `scripts/error-detector.sh` | PostToolUse (Bash) | Triggers on command errors | + +See `references/hooks-setup.md` for detailed configuration and troubleshooting. + +## Automatic Skill Extraction + +When a learning is valuable enough to become a reusable skill, extract it using the provided helper. + +### Skill Extraction Criteria + +A learning qualifies for skill extraction when ANY of these apply: + +| Criterion | Description | +|-----------|-------------| +| **Recurring** | Has `See Also` links to 2+ similar issues | +| **Verified** | Status is `resolved` with working fix | +| **Non-obvious** | Required actual debugging/investigation to discover | +| **Broadly applicable** | Not project-specific; useful across codebases | +| **User-flagged** | User says "save this as a skill" or similar | + +### Extraction Workflow + +1. **Identify candidate**: Learning meets extraction criteria +2. **Run helper** (or create manually): + ```bash + ./skills/self-improvement/scripts/extract-skill.sh skill-name --dry-run + ./skills/self-improvement/scripts/extract-skill.sh skill-name + ``` +3. **Customize SKILL.md**: Fill in template with learning content +4. **Update learning**: Set status to `promoted_to_skill`, add `Skill-Path` +5. **Verify**: Read skill in fresh session to ensure it's self-contained + +### Manual Extraction + +If you prefer manual creation: + +1. Create `skills//SKILL.md` +2. Use template from `assets/SKILL-TEMPLATE.md` +3. Follow [Agent Skills spec](https://agentskills.io/specification): + - YAML frontmatter with `name` and `description` + - Name must match folder name + - No README.md inside skill folder + +### Extraction Detection Triggers + +Watch for these signals that a learning should become a skill: + +**In conversation:** +- "Save this as a skill" +- "I keep running into this" +- "This would be useful for other projects" +- "Remember this pattern" + +**In learning entries:** +- Multiple `See Also` links (recurring issue) +- High priority + resolved status +- Category: `best_practice` with broad applicability +- User feedback praising the solution + +### Skill Quality Gates + +Before extraction, verify: + +- [ ] Solution is tested and working +- [ ] Description is clear without original context +- [ ] Code examples are self-contained +- [ ] No project-specific hardcoded values +- [ ] Follows skill naming conventions (lowercase, hyphens) + +## Multi-Agent Support + +This skill works across different AI coding agents with agent-specific activation. + +### Claude Code + +**Activation**: Hooks (UserPromptSubmit, PostToolUse) +**Setup**: `.claude/settings.json` with hook configuration +**Detection**: Automatic via hook scripts + +### Codex CLI + +**Activation**: Hooks (same pattern as Claude Code) +**Setup**: `.codex/settings.json` with hook configuration +**Detection**: Automatic via hook scripts + +### GitHub Copilot + +**Activation**: Manual (no hook support) +**Setup**: Add to `.github/copilot-instructions.md`: + +```markdown +## Self-Improvement + +After solving non-obvious issues, consider logging to `.learnings/`: +1. Use format from self-improvement skill +2. Link related entries with See Also +3. Promote high-value learnings to skills + +Ask in chat: "Should I log this as a learning?" +``` + +**Detection**: Manual review at session end + +### OpenClaw + +**Activation**: Workspace injection + inter-agent messaging +**Setup**: See "OpenClaw Setup" section above +**Detection**: Via session tools and workspace files + +### Agent-Agnostic Guidance + +Regardless of agent, apply self-improvement when you: + +1. **Discover something non-obvious** - solution wasn't immediate +2. **Correct yourself** - initial approach was wrong +3. **Learn project conventions** - discovered undocumented patterns +4. **Hit unexpected errors** - especially if diagnosis was difficult +5. **Find better approaches** - improved on your original solution + +### Copilot Chat Integration + +For Copilot users, add this to your prompts when relevant: + +> After completing this task, evaluate if any learnings should be logged to `.learnings/` using the self-improvement skill format. + +Or use quick prompts: +- "Log this to learnings" +- "Create a skill from this solution" +- "Check .learnings/ for related issues" diff --git a/skills/self-improving-agent/_meta.json b/skills/self-improving-agent/_meta.json new file mode 100644 index 0000000..254b9f7 --- /dev/null +++ b/skills/self-improving-agent/_meta.json @@ -0,0 +1,6 @@ +{ + "ownerId": "kn70cjr952qdec1nx70zs6wefn7ynq2t", + "slug": "self-improving-agent", + "version": "1.0.11", + "publishedAt": 1771777713337 +} \ No newline at end of file diff --git a/skills/self-improving-agent/assets/LEARNINGS.md b/skills/self-improving-agent/assets/LEARNINGS.md new file mode 100644 index 0000000..6993f9b --- /dev/null +++ b/skills/self-improving-agent/assets/LEARNINGS.md @@ -0,0 +1,45 @@ +# Learnings + +Corrections, insights, and knowledge gaps captured during development. + +**Categories**: correction | insight | knowledge_gap | best_practice +**Areas**: frontend | backend | infra | tests | docs | config +**Statuses**: pending | in_progress | resolved | wont_fix | promoted | promoted_to_skill + +## Status Definitions + +| Status | Meaning | +|--------|---------| +| `pending` | Not yet addressed | +| `in_progress` | Actively being worked on | +| `resolved` | Issue fixed or knowledge integrated | +| `wont_fix` | Decided not to address (reason in Resolution) | +| `promoted` | Elevated to CLAUDE.md, AGENTS.md, or copilot-instructions.md | +| `promoted_to_skill` | Extracted as a reusable skill | + +## Skill Extraction Fields + +When a learning is promoted to a skill, add these fields: + +```markdown +**Status**: promoted_to_skill +**Skill-Path**: skills/skill-name +``` + +Example: +```markdown +## [LRN-20250115-001] best_practice + +**Logged**: 2025-01-15T10:00:00Z +**Priority**: high +**Status**: promoted_to_skill +**Skill-Path**: skills/docker-m1-fixes +**Area**: infra + +### Summary +Docker build fails on Apple Silicon due to platform mismatch +... +``` + +--- + diff --git a/skills/self-improving-agent/assets/SKILL-TEMPLATE.md b/skills/self-improving-agent/assets/SKILL-TEMPLATE.md new file mode 100644 index 0000000..0162134 --- /dev/null +++ b/skills/self-improving-agent/assets/SKILL-TEMPLATE.md @@ -0,0 +1,177 @@ +# Skill Template + +Template for creating skills extracted from learnings. Copy and customize. + +--- + +## SKILL.md Template + +```markdown +--- +name: skill-name-here +description: "Concise description of when and why to use this skill. Include trigger conditions." +--- + +# Skill Name + +Brief introduction explaining the problem this skill solves and its origin. + +## Quick Reference + +| Situation | Action | +|-----------|--------| +| [Trigger 1] | [Action 1] | +| [Trigger 2] | [Action 2] | + +## Background + +Why this knowledge matters. What problems it prevents. Context from the original learning. + +## Solution + +### Step-by-Step + +1. First step with code or command +2. Second step +3. Verification step + +### Code Example + +\`\`\`language +// Example code demonstrating the solution +\`\`\` + +## Common Variations + +- **Variation A**: Description and how to handle +- **Variation B**: Description and how to handle + +## Gotchas + +- Warning or common mistake #1 +- Warning or common mistake #2 + +## Related + +- Link to related documentation +- Link to related skill + +## Source + +Extracted from learning entry. +- **Learning ID**: LRN-YYYYMMDD-XXX +- **Original Category**: correction | insight | knowledge_gap | best_practice +- **Extraction Date**: YYYY-MM-DD +``` + +--- + +## Minimal Template + +For simple skills that don't need all sections: + +```markdown +--- +name: skill-name-here +description: "What this skill does and when to use it." +--- + +# Skill Name + +[Problem statement in one sentence] + +## Solution + +[Direct solution with code/commands] + +## Source + +- Learning ID: LRN-YYYYMMDD-XXX +``` + +--- + +## Template with Scripts + +For skills that include executable helpers: + +```markdown +--- +name: skill-name-here +description: "What this skill does and when to use it." +--- + +# Skill Name + +[Introduction] + +## Quick Reference + +| Command | Purpose | +|---------|---------| +| `./scripts/helper.sh` | [What it does] | +| `./scripts/validate.sh` | [What it does] | + +## Usage + +### Automated (Recommended) + +\`\`\`bash +./skills/skill-name/scripts/helper.sh [args] +\`\`\` + +### Manual Steps + +1. Step one +2. Step two + +## Scripts + +| Script | Description | +|--------|-------------| +| `scripts/helper.sh` | Main utility | +| `scripts/validate.sh` | Validation checker | + +## Source + +- Learning ID: LRN-YYYYMMDD-XXX +``` + +--- + +## Naming Conventions + +- **Skill name**: lowercase, hyphens for spaces + - Good: `docker-m1-fixes`, `api-timeout-patterns` + - Bad: `Docker_M1_Fixes`, `APITimeoutPatterns` + +- **Description**: Start with action verb, mention trigger + - Good: "Handles Docker build failures on Apple Silicon. Use when builds fail with platform mismatch." + - Bad: "Docker stuff" + +- **Files**: + - `SKILL.md` - Required, main documentation + - `scripts/` - Optional, executable code + - `references/` - Optional, detailed docs + - `assets/` - Optional, templates + +--- + +## Extraction Checklist + +Before creating a skill from a learning: + +- [ ] Learning is verified (status: resolved) +- [ ] Solution is broadly applicable (not one-off) +- [ ] Content is complete (has all needed context) +- [ ] Name follows conventions +- [ ] Description is concise but informative +- [ ] Quick Reference table is actionable +- [ ] Code examples are tested +- [ ] Source learning ID is recorded + +After creating: + +- [ ] Update original learning with `promoted_to_skill` status +- [ ] Add `Skill-Path: skills/skill-name` to learning metadata +- [ ] Test skill by reading it in a fresh session diff --git a/skills/self-improving-agent/hooks/openclaw/HOOK.md b/skills/self-improving-agent/hooks/openclaw/HOOK.md new file mode 100644 index 0000000..df67405 --- /dev/null +++ b/skills/self-improving-agent/hooks/openclaw/HOOK.md @@ -0,0 +1,23 @@ +--- +name: self-improvement +description: "Injects self-improvement reminder during agent bootstrap" +metadata: {"openclaw":{"emoji":"🧠","events":["agent:bootstrap"]}} +--- + +# Self-Improvement Hook + +Injects a reminder to evaluate learnings during agent bootstrap. + +## What It Does + +- Fires on `agent:bootstrap` (before workspace files are injected) +- Adds a reminder block to check `.learnings/` for relevant entries +- Prompts the agent to log corrections, errors, and discoveries + +## Configuration + +No configuration needed. Enable with: + +```bash +openclaw hooks enable self-improvement +``` diff --git a/skills/self-improving-agent/hooks/openclaw/handler.js b/skills/self-improving-agent/hooks/openclaw/handler.js new file mode 100644 index 0000000..73278ea --- /dev/null +++ b/skills/self-improving-agent/hooks/openclaw/handler.js @@ -0,0 +1,56 @@ +/** + * Self-Improvement Hook for OpenClaw + * + * Injects a reminder to evaluate learnings during agent bootstrap. + * Fires on agent:bootstrap event before workspace files are injected. + */ + +const REMINDER_CONTENT = ` +## Self-Improvement Reminder + +After completing tasks, evaluate if any learnings should be captured: + +**Log when:** +- User corrects you → \`.learnings/LEARNINGS.md\` +- Command/operation fails → \`.learnings/ERRORS.md\` +- User wants missing capability → \`.learnings/FEATURE_REQUESTS.md\` +- You discover your knowledge was wrong → \`.learnings/LEARNINGS.md\` +- You find a better approach → \`.learnings/LEARNINGS.md\` + +**Promote when pattern is proven:** +- Behavioral patterns → \`SOUL.md\` +- Workflow improvements → \`AGENTS.md\` +- Tool gotchas → \`TOOLS.md\` + +Keep entries simple: date, title, what happened, what to do differently. +`.trim(); + +const handler = async (event) => { + // Safety checks for event structure + if (!event || typeof event !== 'object') { + return; + } + + // Only handle agent:bootstrap events + if (event.type !== 'agent' || event.action !== 'bootstrap') { + return; + } + + // Safety check for context + if (!event.context || typeof event.context !== 'object') { + return; + } + + // Inject the reminder as a virtual bootstrap file + // Check that bootstrapFiles is an array before pushing + if (Array.isArray(event.context.bootstrapFiles)) { + event.context.bootstrapFiles.push({ + path: 'SELF_IMPROVEMENT_REMINDER.md', + content: REMINDER_CONTENT, + virtual: true, + }); + } +}; + +module.exports = handler; +module.exports.default = handler; diff --git a/skills/self-improving-agent/hooks/openclaw/handler.ts b/skills/self-improving-agent/hooks/openclaw/handler.ts new file mode 100644 index 0000000..9ec23f3 --- /dev/null +++ b/skills/self-improving-agent/hooks/openclaw/handler.ts @@ -0,0 +1,62 @@ +/** + * Self-Improvement Hook for OpenClaw + * + * Injects a reminder to evaluate learnings during agent bootstrap. + * Fires on agent:bootstrap event before workspace files are injected. + */ + +import type { HookHandler } from 'openclaw/hooks'; + +const REMINDER_CONTENT = `## Self-Improvement Reminder + +After completing tasks, evaluate if any learnings should be captured: + +**Log when:** +- User corrects you → \`.learnings/LEARNINGS.md\` +- Command/operation fails → \`.learnings/ERRORS.md\` +- User wants missing capability → \`.learnings/FEATURE_REQUESTS.md\` +- You discover your knowledge was wrong → \`.learnings/LEARNINGS.md\` +- You find a better approach → \`.learnings/LEARNINGS.md\` + +**Promote when pattern is proven:** +- Behavioral patterns → \`SOUL.md\` +- Workflow improvements → \`AGENTS.md\` +- Tool gotchas → \`TOOLS.md\` + +Keep entries simple: date, title, what happened, what to do differently.`; + +const handler: HookHandler = async (event) => { + // Safety checks for event structure + if (!event || typeof event !== 'object') { + return; + } + + // Only handle agent:bootstrap events + if (event.type !== 'agent' || event.action !== 'bootstrap') { + return; + } + + // Safety check for context + if (!event.context || typeof event.context !== 'object') { + return; + } + + // Skip sub-agent sessions to avoid bootstrap issues + // Sub-agents have sessionKey patterns like "agent:main:subagent:..." + const sessionKey = event.sessionKey || ''; + if (sessionKey.includes(':subagent:')) { + return; + } + + // Inject the reminder as a virtual bootstrap file + // Check that bootstrapFiles is an array before pushing + if (Array.isArray(event.context.bootstrapFiles)) { + event.context.bootstrapFiles.push({ + path: 'SELF_IMPROVEMENT_REMINDER.md', + content: REMINDER_CONTENT, + virtual: true, + }); + } +}; + +export default handler; diff --git a/skills/self-improving-agent/references/examples.md b/skills/self-improving-agent/references/examples.md new file mode 100644 index 0000000..1c1db15 --- /dev/null +++ b/skills/self-improving-agent/references/examples.md @@ -0,0 +1,374 @@ +# Entry Examples + +Concrete examples of well-formatted entries with all fields. + +## Learning: Correction + +```markdown +## [LRN-20250115-001] correction + +**Logged**: 2025-01-15T10:30:00Z +**Priority**: high +**Status**: pending +**Area**: tests + +### Summary +Incorrectly assumed pytest fixtures are scoped to function by default + +### Details +When writing test fixtures, I assumed all fixtures were function-scoped. +User corrected that while function scope is the default, the codebase +convention uses module-scoped fixtures for database connections to +improve test performance. + +### Suggested Action +When creating fixtures that involve expensive setup (DB, network), +check existing fixtures for scope patterns before defaulting to function scope. + +### Metadata +- Source: user_feedback +- Related Files: tests/conftest.py +- Tags: pytest, testing, fixtures + +--- +``` + +## Learning: Knowledge Gap (Resolved) + +```markdown +## [LRN-20250115-002] knowledge_gap + +**Logged**: 2025-01-15T14:22:00Z +**Priority**: medium +**Status**: resolved +**Area**: config + +### Summary +Project uses pnpm not npm for package management + +### Details +Attempted to run `npm install` but project uses pnpm workspaces. +Lock file is `pnpm-lock.yaml`, not `package-lock.json`. + +### Suggested Action +Check for `pnpm-lock.yaml` or `pnpm-workspace.yaml` before assuming npm. +Use `pnpm install` for this project. + +### Metadata +- Source: error +- Related Files: pnpm-lock.yaml, pnpm-workspace.yaml +- Tags: package-manager, pnpm, setup + +### Resolution +- **Resolved**: 2025-01-15T14:30:00Z +- **Commit/PR**: N/A - knowledge update +- **Notes**: Added to CLAUDE.md for future reference + +--- +``` + +## Learning: Promoted to CLAUDE.md + +```markdown +## [LRN-20250115-003] best_practice + +**Logged**: 2025-01-15T16:00:00Z +**Priority**: high +**Status**: promoted +**Promoted**: CLAUDE.md +**Area**: backend + +### Summary +API responses must include correlation ID from request headers + +### Details +All API responses should echo back the X-Correlation-ID header from +the request. This is required for distributed tracing. Responses +without this header break the observability pipeline. + +### Suggested Action +Always include correlation ID passthrough in API handlers. + +### Metadata +- Source: user_feedback +- Related Files: src/middleware/correlation.ts +- Tags: api, observability, tracing + +--- +``` + +## Learning: Promoted to AGENTS.md + +```markdown +## [LRN-20250116-001] best_practice + +**Logged**: 2025-01-16T09:00:00Z +**Priority**: high +**Status**: promoted +**Promoted**: AGENTS.md +**Area**: backend + +### Summary +Must regenerate API client after OpenAPI spec changes + +### Details +When modifying API endpoints, the TypeScript client must be regenerated. +Forgetting this causes type mismatches that only appear at runtime. +The generate script also runs validation. + +### Suggested Action +Add to agent workflow: after any API changes, run `pnpm run generate:api`. + +### Metadata +- Source: error +- Related Files: openapi.yaml, src/client/api.ts +- Tags: api, codegen, typescript + +--- +``` + +## Error Entry + +```markdown +## [ERR-20250115-A3F] docker_build + +**Logged**: 2025-01-15T09:15:00Z +**Priority**: high +**Status**: pending +**Area**: infra + +### Summary +Docker build fails on M1 Mac due to platform mismatch + +### Error +``` +error: failed to solve: python:3.11-slim: no match for platform linux/arm64 +``` + +### Context +- Command: `docker build -t myapp .` +- Dockerfile uses `FROM python:3.11-slim` +- Running on Apple Silicon (M1/M2) + +### Suggested Fix +Add platform flag: `docker build --platform linux/amd64 -t myapp .` +Or update Dockerfile: `FROM --platform=linux/amd64 python:3.11-slim` + +### Metadata +- Reproducible: yes +- Related Files: Dockerfile + +--- +``` + +## Error Entry: Recurring Issue + +```markdown +## [ERR-20250120-B2C] api_timeout + +**Logged**: 2025-01-20T11:30:00Z +**Priority**: critical +**Status**: pending +**Area**: backend + +### Summary +Third-party payment API timeout during checkout + +### Error +``` +TimeoutError: Request to payments.example.com timed out after 30000ms +``` + +### Context +- Command: POST /api/checkout +- Timeout set to 30s +- Occurs during peak hours (lunch, evening) + +### Suggested Fix +Implement retry with exponential backoff. Consider circuit breaker pattern. + +### Metadata +- Reproducible: yes (during peak hours) +- Related Files: src/services/payment.ts +- See Also: ERR-20250115-X1Y, ERR-20250118-Z3W + +--- +``` + +## Feature Request + +```markdown +## [FEAT-20250115-001] export_to_csv + +**Logged**: 2025-01-15T16:45:00Z +**Priority**: medium +**Status**: pending +**Area**: backend + +### Requested Capability +Export analysis results to CSV format + +### User Context +User runs weekly reports and needs to share results with non-technical +stakeholders in Excel. Currently copies output manually. + +### Complexity Estimate +simple + +### Suggested Implementation +Add `--output csv` flag to the analyze command. Use standard csv module. +Could extend existing `--output json` pattern. + +### Metadata +- Frequency: recurring +- Related Features: analyze command, json output + +--- +``` + +## Feature Request: Resolved + +```markdown +## [FEAT-20250110-002] dark_mode + +**Logged**: 2025-01-10T14:00:00Z +**Priority**: low +**Status**: resolved +**Area**: frontend + +### Requested Capability +Dark mode support for the dashboard + +### User Context +User works late hours and finds the bright interface straining. +Several other users have mentioned this informally. + +### Complexity Estimate +medium + +### Suggested Implementation +Use CSS variables for colors. Add toggle in user settings. +Consider system preference detection. + +### Metadata +- Frequency: recurring +- Related Features: user settings, theme system + +### Resolution +- **Resolved**: 2025-01-18T16:00:00Z +- **Commit/PR**: #142 +- **Notes**: Implemented with system preference detection and manual toggle + +--- +``` + +## Learning: Promoted to Skill + +```markdown +## [LRN-20250118-001] best_practice + +**Logged**: 2025-01-18T11:00:00Z +**Priority**: high +**Status**: promoted_to_skill +**Skill-Path**: skills/docker-m1-fixes +**Area**: infra + +### Summary +Docker build fails on Apple Silicon due to platform mismatch + +### Details +When building Docker images on M1/M2 Macs, the build fails because +the base image doesn't have an ARM64 variant. This is a common issue +that affects many developers. + +### Suggested Action +Add `--platform linux/amd64` to docker build command, or use +`FROM --platform=linux/amd64` in Dockerfile. + +### Metadata +- Source: error +- Related Files: Dockerfile +- Tags: docker, arm64, m1, apple-silicon +- See Also: ERR-20250115-A3F, ERR-20250117-B2D + +--- +``` + +## Extracted Skill Example + +When the above learning is extracted as a skill, it becomes: + +**File**: `skills/docker-m1-fixes/SKILL.md` + +```markdown +--- +name: docker-m1-fixes +description: "Fixes Docker build failures on Apple Silicon (M1/M2). Use when docker build fails with platform mismatch errors." +--- + +# Docker M1 Fixes + +Solutions for Docker build issues on Apple Silicon Macs. + +## Quick Reference + +| Error | Fix | +|-------|-----| +| `no match for platform linux/arm64` | Add `--platform linux/amd64` to build | +| Image runs but crashes | Use emulation or find ARM-compatible base | + +## The Problem + +Many Docker base images don't have ARM64 variants. When building on +Apple Silicon (M1/M2/M3), Docker attempts to pull ARM64 images by +default, causing platform mismatch errors. + +## Solutions + +### Option 1: Build Flag (Recommended) + +Add platform flag to your build command: + +\`\`\`bash +docker build --platform linux/amd64 -t myapp . +\`\`\` + +### Option 2: Dockerfile Modification + +Specify platform in the FROM instruction: + +\`\`\`dockerfile +FROM --platform=linux/amd64 python:3.11-slim +\`\`\` + +### Option 3: Docker Compose + +Add platform to your service: + +\`\`\`yaml +services: + app: + platform: linux/amd64 + build: . +\`\`\` + +## Trade-offs + +| Approach | Pros | Cons | +|----------|------|------| +| Build flag | No file changes | Must remember flag | +| Dockerfile | Explicit, versioned | Affects all builds | +| Compose | Convenient for dev | Requires compose | + +## Performance Note + +Running AMD64 images on ARM64 uses Rosetta 2 emulation. This works +for development but may be slower. For production, find ARM-native +alternatives when possible. + +## Source + +- Learning ID: LRN-20250118-001 +- Category: best_practice +- Extraction Date: 2025-01-18 +``` diff --git a/skills/self-improving-agent/references/hooks-setup.md b/skills/self-improving-agent/references/hooks-setup.md new file mode 100644 index 0000000..08b5dd1 --- /dev/null +++ b/skills/self-improving-agent/references/hooks-setup.md @@ -0,0 +1,223 @@ +# Hook Setup Guide + +Configure automatic self-improvement triggers for AI coding agents. + +## Overview + +Hooks enable proactive learning capture by injecting reminders at key moments: +- **UserPromptSubmit**: Reminder after each prompt to evaluate learnings +- **PostToolUse (Bash)**: Error detection when commands fail + +## Claude Code Setup + +### Option 1: Project-Level Configuration + +Create `.claude/settings.json` in your project root: + +```json +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "./skills/self-improvement/scripts/activator.sh" + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "./skills/self-improvement/scripts/error-detector.sh" + } + ] + } + ] + } +} +``` + +### Option 2: User-Level Configuration + +Add to `~/.claude/settings.json` for global activation: + +```json +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "~/.claude/skills/self-improvement/scripts/activator.sh" + } + ] + } + ] + } +} +``` + +### Minimal Setup (Activator Only) + +For lower overhead, use only the UserPromptSubmit hook: + +```json +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "./skills/self-improvement/scripts/activator.sh" + } + ] + } + ] + } +} +``` + +## Codex CLI Setup + +Codex uses the same hook system as Claude Code. Create `.codex/settings.json`: + +```json +{ + "hooks": { + "UserPromptSubmit": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "./skills/self-improvement/scripts/activator.sh" + } + ] + } + ] + } +} +``` + +## GitHub Copilot Setup + +Copilot doesn't support hooks directly. Instead, add guidance to `.github/copilot-instructions.md`: + +```markdown +## Self-Improvement + +After completing tasks that involved: +- Debugging non-obvious issues +- Discovering workarounds +- Learning project-specific patterns +- Resolving unexpected errors + +Consider logging the learning to `.learnings/` using the format from the self-improvement skill. + +For high-value learnings that would benefit other sessions, consider skill extraction. +``` + +## Verification + +### Test Activator Hook + +1. Enable the hook configuration +2. Start a new Claude Code session +3. Send any prompt +4. Verify you see `` in the context + +### Test Error Detector Hook + +1. Enable PostToolUse hook for Bash +2. Run a command that fails: `ls /nonexistent/path` +3. Verify you see `` reminder + +### Dry Run Extract Script + +```bash +./skills/self-improvement/scripts/extract-skill.sh test-skill --dry-run +``` + +Expected output shows the skill scaffold that would be created. + +## Troubleshooting + +### Hook Not Triggering + +1. **Check script permissions**: `chmod +x scripts/*.sh` +2. **Verify path**: Use absolute paths or paths relative to project root +3. **Check settings location**: Project vs user-level settings +4. **Restart session**: Hooks are loaded at session start + +### Permission Denied + +```bash +chmod +x ./skills/self-improvement/scripts/activator.sh +chmod +x ./skills/self-improvement/scripts/error-detector.sh +chmod +x ./skills/self-improvement/scripts/extract-skill.sh +``` + +### Script Not Found + +If using relative paths, ensure you're in the correct directory or use absolute paths: + +```json +{ + "command": "/absolute/path/to/skills/self-improvement/scripts/activator.sh" +} +``` + +### Too Much Overhead + +If the activator feels intrusive: + +1. **Use minimal setup**: Only UserPromptSubmit, skip PostToolUse +2. **Add matcher filter**: Only trigger for certain prompts: + +```json +{ + "matcher": "fix|debug|error|issue", + "hooks": [...] +} +``` + +## Hook Output Budget + +The activator is designed to be lightweight: +- **Target**: ~50-100 tokens per activation +- **Content**: Structured reminder, not verbose instructions +- **Format**: XML tags for easy parsing + +If you need to reduce overhead further, you can edit `activator.sh` to output less text. + +## Security Considerations + +- Hook scripts run with the same permissions as Claude Code +- Scripts only output text; they don't modify files or run commands +- Error detector reads `CLAUDE_TOOL_OUTPUT` environment variable +- All scripts are opt-in (you must configure them explicitly) + +## Disabling Hooks + +To temporarily disable without removing configuration: + +1. **Comment out in settings**: +```json +{ + "hooks": { + // "UserPromptSubmit": [...] + } +} +``` + +2. **Or delete the settings file**: Hooks won't run without configuration diff --git a/skills/self-improving-agent/references/openclaw-integration.md b/skills/self-improving-agent/references/openclaw-integration.md new file mode 100644 index 0000000..09f0193 --- /dev/null +++ b/skills/self-improving-agent/references/openclaw-integration.md @@ -0,0 +1,248 @@ +# OpenClaw Integration + +Complete setup and usage guide for integrating the self-improvement skill with OpenClaw. + +## Overview + +OpenClaw uses workspace-based prompt injection combined with event-driven hooks. Context is injected from workspace files at session start, and hooks can trigger on lifecycle events. + +## Workspace Structure + +``` +~/.openclaw/ +├── workspace/ # Working directory +│ ├── AGENTS.md # Multi-agent coordination patterns +│ ├── SOUL.md # Behavioral guidelines and personality +│ ├── TOOLS.md # Tool capabilities and gotchas +│ ├── MEMORY.md # Long-term memory (main session only) +│ └── memory/ # Daily memory files +│ └── YYYY-MM-DD.md +├── skills/ # Installed skills +│ └── / +│ └── SKILL.md +└── hooks/ # Custom hooks + └── / + ├── HOOK.md + └── handler.ts +``` + +## Quick Setup + +### 1. Install the Skill + +```bash +clawdhub install self-improving-agent +``` + +Or copy manually: + +```bash +cp -r self-improving-agent ~/.openclaw/skills/ +``` + +### 2. Install the Hook (Optional) + +Copy the hook to OpenClaw's hooks directory: + +```bash +cp -r hooks/openclaw ~/.openclaw/hooks/self-improvement +``` + +Enable the hook: + +```bash +openclaw hooks enable self-improvement +``` + +### 3. Create Learning Files + +Create the `.learnings/` directory in your workspace: + +```bash +mkdir -p ~/.openclaw/workspace/.learnings +``` + +Or in the skill directory: + +```bash +mkdir -p ~/.openclaw/skills/self-improving-agent/.learnings +``` + +## Injected Prompt Files + +### AGENTS.md + +Purpose: Multi-agent workflows and delegation patterns. + +```markdown +# Agent Coordination + +## Delegation Rules +- Use explore agent for open-ended codebase questions +- Spawn sub-agents for long-running tasks +- Use sessions_send for cross-session communication + +## Session Handoff +When delegating to another session: +1. Provide full context in the handoff message +2. Include relevant file paths +3. Specify expected output format +``` + +### SOUL.md + +Purpose: Behavioral guidelines and communication style. + +```markdown +# Behavioral Guidelines + +## Communication Style +- Be direct and concise +- Avoid unnecessary caveats and disclaimers +- Use technical language appropriate to context + +## Error Handling +- Admit mistakes promptly +- Provide corrected information immediately +- Log significant errors to learnings +``` + +### TOOLS.md + +Purpose: Tool capabilities, integration gotchas, local configuration. + +```markdown +# Tool Knowledge + +## Self-Improvement Skill +Log learnings to `.learnings/` for continuous improvement. + +## Local Tools +- Document tool-specific gotchas here +- Note authentication requirements +- Track integration quirks +``` + +## Learning Workflow + +### Capturing Learnings + +1. **In-session**: Log to `.learnings/` as usual +2. **Cross-session**: Promote to workspace files + +### Promotion Decision Tree + +``` +Is the learning project-specific? +├── Yes → Keep in .learnings/ +└── No → Is it behavioral/style-related? + ├── Yes → Promote to SOUL.md + └── No → Is it tool-related? + ├── Yes → Promote to TOOLS.md + └── No → Promote to AGENTS.md (workflow) +``` + +### Promotion Format Examples + +**From learning:** +> Git push to GitHub fails without auth configured - triggers desktop prompt + +**To TOOLS.md:** +```markdown +## Git +- Don't push without confirming auth is configured +- Use `gh auth status` to check GitHub CLI auth +``` + +## Inter-Agent Communication + +OpenClaw provides tools for cross-session communication: + +### sessions_list + +View active and recent sessions: +``` +sessions_list(activeMinutes=30, messageLimit=3) +``` + +### sessions_history + +Read transcript from another session: +``` +sessions_history(sessionKey="session-id", limit=50) +``` + +### sessions_send + +Send message to another session: +``` +sessions_send(sessionKey="session-id", message="Learning: API requires X-Custom-Header") +``` + +### sessions_spawn + +Spawn a background sub-agent: +``` +sessions_spawn(task="Research X and report back", label="research") +``` + +## Available Hook Events + +| Event | When It Fires | +|-------|---------------| +| `agent:bootstrap` | Before workspace files inject | +| `command:new` | When `/new` command issued | +| `command:reset` | When `/reset` command issued | +| `command:stop` | When `/stop` command issued | +| `gateway:startup` | When gateway starts | + +## Detection Triggers + +### Standard Triggers +- User corrections ("No, that's wrong...") +- Command failures (non-zero exit codes) +- API errors +- Knowledge gaps + +### OpenClaw-Specific Triggers + +| Trigger | Action | +|---------|--------| +| Tool call error | Log to TOOLS.md with tool name | +| Session handoff confusion | Log to AGENTS.md with delegation pattern | +| Model behavior surprise | Log to SOUL.md with expected vs actual | +| Skill issue | Log to .learnings/ or report upstream | + +## Verification + +Check hook is registered: + +```bash +openclaw hooks list +``` + +Check skill is loaded: + +```bash +openclaw status +``` + +## Troubleshooting + +### Hook not firing + +1. Ensure hooks enabled in config +2. Restart gateway after config changes +3. Check gateway logs for errors + +### Learnings not persisting + +1. Verify `.learnings/` directory exists +2. Check file permissions +3. Ensure workspace path is configured correctly + +### Skill not loading + +1. Check skill is in skills directory +2. Verify SKILL.md has correct frontmatter +3. Run `openclaw status` to see loaded skills diff --git a/skills/self-improving-agent/scripts/activator.sh b/skills/self-improving-agent/scripts/activator.sh new file mode 100644 index 0000000..29eec22 --- /dev/null +++ b/skills/self-improving-agent/scripts/activator.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Self-Improvement Activator Hook +# Triggers on UserPromptSubmit to remind Claude about learning capture +# Keep output minimal (~50-100 tokens) to minimize overhead + +set -e + +# Output reminder as system context +cat << 'EOF' + +After completing this task, evaluate if extractable knowledge emerged: +- Non-obvious solution discovered through investigation? +- Workaround for unexpected behavior? +- Project-specific pattern learned? +- Error required debugging to resolve? + +If yes: Log to .learnings/ using the self-improvement skill format. +If high-value (recurring, broadly applicable): Consider skill extraction. + +EOF diff --git a/skills/self-improving-agent/scripts/error-detector.sh b/skills/self-improving-agent/scripts/error-detector.sh new file mode 100644 index 0000000..3c310dd --- /dev/null +++ b/skills/self-improving-agent/scripts/error-detector.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Self-Improvement Error Detector Hook +# Triggers on PostToolUse for Bash to detect command failures +# Reads CLAUDE_TOOL_OUTPUT environment variable + +set -e + +# Check if tool output indicates an error +# CLAUDE_TOOL_OUTPUT contains the result of the tool execution +OUTPUT="${CLAUDE_TOOL_OUTPUT:-}" + +# Patterns indicating errors (case-insensitive matching) +ERROR_PATTERNS=( + "error:" + "Error:" + "ERROR:" + "failed" + "FAILED" + "command not found" + "No such file" + "Permission denied" + "fatal:" + "Exception" + "Traceback" + "npm ERR!" + "ModuleNotFoundError" + "SyntaxError" + "TypeError" + "exit code" + "non-zero" +) + +# Check if output contains any error pattern +contains_error=false +for pattern in "${ERROR_PATTERNS[@]}"; do + if [[ "$OUTPUT" == *"$pattern"* ]]; then + contains_error=true + break + fi +done + +# Only output reminder if error detected +if [ "$contains_error" = true ]; then + cat << 'EOF' + +A command error was detected. Consider logging this to .learnings/ERRORS.md if: +- The error was unexpected or non-obvious +- It required investigation to resolve +- It might recur in similar contexts +- The solution could benefit future sessions + +Use the self-improvement skill format: [ERR-YYYYMMDD-XXX] + +EOF +fi diff --git a/skills/self-improving-agent/scripts/extract-skill.sh b/skills/self-improving-agent/scripts/extract-skill.sh new file mode 100644 index 0000000..ccae55a --- /dev/null +++ b/skills/self-improving-agent/scripts/extract-skill.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# Skill Extraction Helper +# Creates a new skill from a learning entry +# Usage: ./extract-skill.sh [--dry-run] + +set -e + +# Configuration +SKILLS_DIR="./skills" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +usage() { + cat << EOF +Usage: $(basename "$0") [options] + +Create a new skill from a learning entry. + +Arguments: + skill-name Name of the skill (lowercase, hyphens for spaces) + +Options: + --dry-run Show what would be created without creating files + --output-dir Relative output directory under current path (default: ./skills) + -h, --help Show this help message + +Examples: + $(basename "$0") docker-m1-fixes + $(basename "$0") api-timeout-patterns --dry-run + $(basename "$0") pnpm-setup --output-dir ./skills/custom + +The skill will be created in: \$SKILLS_DIR// +EOF +} + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Parse arguments +SKILL_NAME="" +DRY_RUN=false + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN=true + shift + ;; + --output-dir) + if [ -z "${2:-}" ] || [[ "${2:-}" == -* ]]; then + log_error "--output-dir requires a relative path argument" + usage + exit 1 + fi + SKILLS_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + if [ -z "$SKILL_NAME" ]; then + SKILL_NAME="$1" + else + log_error "Unexpected argument: $1" + usage + exit 1 + fi + shift + ;; + esac +done + +# Validate skill name +if [ -z "$SKILL_NAME" ]; then + log_error "Skill name is required" + usage + exit 1 +fi + +# Validate skill name format (lowercase, hyphens, no spaces) +if ! [[ "$SKILL_NAME" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then + log_error "Invalid skill name format. Use lowercase letters, numbers, and hyphens only." + log_error "Examples: 'docker-fixes', 'api-patterns', 'pnpm-setup'" + exit 1 +fi + +# Validate output path to avoid writes outside current workspace. +if [[ "$SKILLS_DIR" = /* ]]; then + log_error "Output directory must be a relative path under the current directory." + exit 1 +fi + +if [[ "$SKILLS_DIR" =~ (^|/)\.\.(/|$) ]]; then + log_error "Output directory cannot include '..' path segments." + exit 1 +fi + +SKILLS_DIR="${SKILLS_DIR#./}" +SKILLS_DIR="./$SKILLS_DIR" + +SKILL_PATH="$SKILLS_DIR/$SKILL_NAME" + +# Check if skill already exists +if [ -d "$SKILL_PATH" ] && [ "$DRY_RUN" = false ]; then + log_error "Skill already exists: $SKILL_PATH" + log_error "Use a different name or remove the existing skill first." + exit 1 +fi + +# Dry run output +if [ "$DRY_RUN" = true ]; then + log_info "Dry run - would create:" + echo " $SKILL_PATH/" + echo " $SKILL_PATH/SKILL.md" + echo "" + echo "Template content would be:" + echo "---" + cat << TEMPLATE +name: $SKILL_NAME +description: "[TODO: Add a concise description of what this skill does and when to use it]" +--- + +# $(echo "$SKILL_NAME" | sed 's/-/ /g' | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) tolower(substr($i,2))}1') + +[TODO: Brief introduction explaining the skill's purpose] + +## Quick Reference + +| Situation | Action | +|-----------|--------| +| [Trigger condition] | [What to do] | + +## Usage + +[TODO: Detailed usage instructions] + +## Examples + +[TODO: Add concrete examples] + +## Source Learning + +This skill was extracted from a learning entry. +- Learning ID: [TODO: Add original learning ID] +- Original File: .learnings/LEARNINGS.md +TEMPLATE + echo "---" + exit 0 +fi + +# Create skill directory structure +log_info "Creating skill: $SKILL_NAME" + +mkdir -p "$SKILL_PATH" + +# Create SKILL.md from template +cat > "$SKILL_PATH/SKILL.md" << TEMPLATE +--- +name: $SKILL_NAME +description: "[TODO: Add a concise description of what this skill does and when to use it]" +--- + +# $(echo "$SKILL_NAME" | sed 's/-/ /g' | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) tolower(substr($i,2))}1') + +[TODO: Brief introduction explaining the skill's purpose] + +## Quick Reference + +| Situation | Action | +|-----------|--------| +| [Trigger condition] | [What to do] | + +## Usage + +[TODO: Detailed usage instructions] + +## Examples + +[TODO: Add concrete examples] + +## Source Learning + +This skill was extracted from a learning entry. +- Learning ID: [TODO: Add original learning ID] +- Original File: .learnings/LEARNINGS.md +TEMPLATE + +log_info "Created: $SKILL_PATH/SKILL.md" + +# Suggest next steps +echo "" +log_info "Skill scaffold created successfully!" +echo "" +echo "Next steps:" +echo " 1. Edit $SKILL_PATH/SKILL.md" +echo " 2. Fill in the TODO sections with content from your learning" +echo " 3. Add references/ folder if you have detailed documentation" +echo " 4. Add scripts/ folder if you have executable code" +echo " 5. Update the original learning entry with:" +echo " **Status**: promoted_to_skill" +echo " **Skill-Path**: skills/$SKILL_NAME" diff --git a/skills/skill-builder/.clawhub/origin.json b/skills/skill-builder/.clawhub/origin.json new file mode 100644 index 0000000..2409d1d --- /dev/null +++ b/skills/skill-builder/.clawhub/origin.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "registry": "https://clawhub.ai", + "slug": "skill-builder", + "installedVersion": "1.0.5", + "installedAt": 1772328256596 +} diff --git a/skills/skill-builder/SKILL.md b/skills/skill-builder/SKILL.md new file mode 100644 index 0000000..121492b --- /dev/null +++ b/skills/skill-builder/SKILL.md @@ -0,0 +1,104 @@ +--- +name: Skill Builder / Creator +slug: skill-builder +version: 1.0.5 +homepage: https://clawic.com/skills/skill-builder +description: Create high-quality skills with modular structure, progressive disclosure, and token-efficient design. +changelog: Added description examples table, security checklist, and improved traps with fixes +metadata: {"clawdbot":{"emoji":"🛠️","requires":{"bins":[]},"os":["linux","darwin","win32"]}} +--- + +## Setup + +On first use, read `setup.md` for integration guidelines. + +## When to Use + +User wants to create or improve a skill. Agent guides structure, reviews content, and ensures quality. + +## Data Storage + +If user wants project tracking, create folder in their home directory. +See `memory-template.md` for the template structure. + +The agent does NOT create files automatically. Always ask user first. + +## Architecture + +Skills follow this structure: + +``` +skill-name/ +├── SKILL.md # Core instructions (SHORT) +├── [topic].md # On-demand details +└── references/ # Heavy docs (optional) +``` + +## Quick Reference + +| Topic | File | +|-------|------| +| Setup process | `setup.md` | +| Tracking projects | `memory-template.md` | +| Patterns and examples | `patterns.md` | + +## Core Rules + +### 1. SKILL.md Must Be Short +Target 30-50 lines, max 80. Move details to auxiliary files. Every line must justify its token cost. + +### 2. Progressive Disclosure +``` +Level 1: Metadata (name + description) — always loaded +Level 2: SKILL.md body — when skill triggers +Level 3: Auxiliary files — on demand +``` + +### 3. Descriptions Are Critical +One sentence, 15-25 words. Action verb first. Describes capabilities, not triggers. + +| ❌ Wrong | ✅ Right | +|----------|----------| +| "Use when user needs PDFs" | "Process, merge, and extract PDF content" | +| "Helper for Docker" | "Build, deploy, and debug Docker containers" | +| "Git guide" | "Manage branches, resolve conflicts, and automate workflows" | + +See `patterns.md` for more examples. + +### 4. Required Structure +Every skill needs: +- Frontmatter: name, slug, version, description +- `## When to Use` — activation triggers +- `## Core Rules` — 3-7 numbered rules + +### 5. Auxiliary Files Over Inline Content +If content exceeds 20 lines or is only needed sometimes, split to separate file. Reference from Quick Reference table. + +### 6. No Redundancy +Information lives in ONE place. SKILL.md references files, doesn't duplicate content. + +### 7. Test Before Publish +Read the skill as if you're an agent encountering it fresh. Is every instruction clear and necessary? + +## Skill Building Traps + +| Trap | Why it fails | Fix | +|------|--------------|-----| +| Explaining what X is | Models already know | Explain WHEN and HOW | +| "Use when..." in description | Wastes characters | Action verbs only | +| Keyword lists in description | Looks spammy | One clean sentence | +| Templates inline | Bloats SKILL.md | Separate file | +| Vague "observe" instructions | Gets flagged suspicious | Be specific about what data | +| Undeclared file creation | Security flag | Add Data Storage section | + +## Related Skills +Install with `clawhub install ` if user confirms: + +- `skill-manager` — manage installed skills +- `skill-update` — update existing skills +- `skill-test` — test skills locally + +## Feedback + +- If useful: `clawhub star skill-builder` +- Stay updated: `clawhub sync` diff --git a/skills/skill-builder/_meta.json b/skills/skill-builder/_meta.json new file mode 100644 index 0000000..ce000d1 --- /dev/null +++ b/skills/skill-builder/_meta.json @@ -0,0 +1,6 @@ +{ + "ownerId": "kn73vp5rarc3b14rc7wjcw8f8580t5d1", + "slug": "skill-builder", + "version": "1.0.5", + "publishedAt": 1772061099771 +} \ No newline at end of file diff --git a/skills/skill-builder/memory-template.md b/skills/skill-builder/memory-template.md new file mode 100644 index 0000000..042dfec --- /dev/null +++ b/skills/skill-builder/memory-template.md @@ -0,0 +1,43 @@ +# Memory Template — Skill Builder / Creator + +**Optional:** If user wants to track projects, they can create `~/skill-builder/projects.md`. + +Ask user before creating any files. Template: + +```markdown +# Skill Projects + +## Active + +### [skill-name] +- status: drafting | reviewing | ready +- goal: [one sentence] +- files: SKILL.md, setup.md, [others] +- notes: [observations, decisions] +- last: YYYY-MM-DD + +## Completed + +### [skill-name] +- published: YYYY-MM-DD +- version: X.Y.Z +- lessons: [what worked, what to improve] + +--- +*Updated: YYYY-MM-DD* +``` + +## Status Values + +| Value | Meaning | +|-------|---------| +| `drafting` | Writing initial content | +| `reviewing` | Checking structure, testing | +| `ready` | Ready to publish | + +## Usage + +- Add new project when user starts skill +- Update status as work progresses +- Move to Completed after publish +- Capture lessons for future skills diff --git a/skills/skill-builder/patterns.md b/skills/skill-builder/patterns.md new file mode 100644 index 0000000..7708f8f --- /dev/null +++ b/skills/skill-builder/patterns.md @@ -0,0 +1,138 @@ +# Patterns — Skill Builder / Creator + +Common patterns for different skill types. + +## Pattern 1: Memory-Based Skills + +Skills that learn and adapt to user preferences. + +``` +skill/ +├── SKILL.md # Instructions + memory reference +├── setup.md # Integration process +├── memory-template.md # Memory structure +└── [domain].md # Domain details +``` + +**Key elements:** +- Memory structure with status tracking +- Rules for when to update memory +- Integration with user's main memory + +## Pattern 2: Tool Integration Skills + +Skills wrapping external tools or APIs. + +``` +skill/ +├── SKILL.md # Workflow + commands +├── setup.md # Installation verification +├── reference.md # Command reference +└── scripts/ # Helper scripts + └── [tool].sh +``` + +**Key elements:** +- External Endpoints table (required) +- Security & Privacy section +- Script manifests +- Error handling guidance + +## Pattern 3: Domain Expert Skills + +Skills providing specialized knowledge. + +``` +skill/ +├── SKILL.md # Overview + rules +├── setup.md # Minimal +├── memory-template.md # Minimal config +└── references/ + ├── [topic1].md + └── [topic2].md +``` + +**Key elements:** +- Progressive loading of references +- Clear triggers in description +- Core Rules capture expert judgment + +## Pattern 4: Workflow Skills + +Skills guiding multi-step processes. + +``` +skill/ +├── SKILL.md # Process overview +├── setup.md # Prerequisites +├── memory-template.md # Progress tracking +├── phases/ +│ ├── phase1.md +│ └── phase2.md +└── templates/ # Output templates +``` + +**Key elements:** +- Clear phase boundaries +- Progress tracking in memory +- Templates for outputs + +## Description Examples + +### Good Descriptions (copy these patterns) + +| Domain | Description | +|--------|-------------| +| PDF | "Process, merge, and extract PDF content with page manipulation and text extraction." | +| Git | "Manage branches, resolve conflicts, and automate Git workflows with best practices." | +| Docker | "Build, deploy, and debug Docker containers with compose patterns and troubleshooting." | +| API | "Design, document, and test REST APIs with OpenAPI specs and mock servers." | +| Database | "Query, optimize, and migrate databases with schema design and performance tuning." | + +### Bad Descriptions (avoid these) + +| ❌ Bad | Why | +|--------|-----| +| "Use when you need to work with PDFs" | Starts with "Use when" | +| "PDF helper. Triggers: pdf, document, merge" | Multiple sentences, keyword list | +| "A comprehensive guide to Docker—including containers, images, and more" | Em-dash, vague "more" | +| "Helper for Git stuff" | Too vague, "stuff" | + +### Formula + +``` +[Verb], [verb], and [verb] [technology] with [feature], [feature], and [feature]. +``` + +15-25 words. One sentence. No em-dashes (—). No "Use when". + +## Frontmatter Checklist + +```yaml +--- +name: Clear Name # What it is +slug: clear-name # Lowercase, hyphens +version: 1.0.0 # Semver +description: One sentence. # Action verbs. 15-25 words. +--- +``` + +## Quality Checklist + +Before publishing: +- [ ] SKILL.md under 80 lines? +- [ ] Description is one sentence, 15-25 words? +- [ ] All required sections present? +- [ ] No redundancy between files? +- [ ] Core Rules are actionable? +- [ ] Traps are real failure modes? + +## Security Checklist + +Avoid getting flagged as suspicious: +- [ ] No vague words: "silently", "secretly", "automatically" +- [ ] If creating files, add `## Data Storage` section +- [ ] If using APIs, add `## External Endpoints` table +- [ ] If using env vars, declare in metadata requires +- [ ] No "observe", "monitor", "track" without specifying WHAT exactly +- [ ] Always mention "ask user first" for file operations diff --git a/skills/skill-builder/setup.md b/skills/skill-builder/setup.md new file mode 100644 index 0000000..6938d93 --- /dev/null +++ b/skills/skill-builder/setup.md @@ -0,0 +1,53 @@ +# Setup — Skill Builder / Creator + +Reference this file when helping users create skills. + +## Your Role + +Help users create effective skills. Guide them through structure, naming, and best practices. + +## Priority Order + +### 1. Understand the Goal + +Ask: +- "What should this skill help with?" +- "What tasks will it handle?" + +Listen for: domain, triggers, audience (human using agent vs agent-to-agent). + +### 2. Identify the Structure + +Based on their goal, determine: +- Does it need memory? (tracks preferences, history, state) +- Does it call external APIs? +- Does it need scripts for deterministic tasks? +- How much auxiliary content? + +### 3. Guide the Build + +Walk them through: +1. Name and description (critical for discovery) +2. Core Rules (what the agent MUST do) +3. Traps (where models fail) +4. File structure + +## Key Principles to Convey + +**Concise over comprehensive:** +"Models are smart. Only add what they don't already know." + +**Progressive disclosure:** +"Details go in separate files, loaded when needed." + +**Description matters most:** +"This is what agents read to decide if your skill matches their query." + +## When Done + +You're ready when: +- Clear understanding of what the skill does +- Draft structure outlined +- User knows what files they need + +Everything else builds iteratively. diff --git a/test_db_connections.py b/test_db_connections.py new file mode 100644 index 0000000..519dd85 --- /dev/null +++ b/test_db_connections.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +""" +数据库连接测试脚本 +仅用于测试连接和读取基本信息，不进行任何写入操作 +""" + +import sys +import json +import warnings +from urllib.parse import quote_plus + +# 忽略 SSL 警告 +warnings.filterwarnings('ignore', message='Unverified HTTPS request') + +def test_es_connection(host, port, scheme, user, password, description): + """测试 Elasticsearch 连接""" + try: + import requests + from requests.auth import HTTPBasicAuth + + url = f"{scheme}://{host}:{port}" + print(f"\n{'='*60}") + print(f"测试: {description}") + print(f"地址: {url}") + print(f"{'='*60}") + + # 测试基本连接 + response = requests.get( + url, + auth=HTTPBasicAuth(user, password), + verify=False, # 忽略 SSL 证书验证（测试环境） + timeout=10 + ) + + if response.status_code == 200: + info = response.json() + print(f"✅ 连接成功!") + print(f" 集群名称: {info.get('cluster_name', 'N/A')}") + print(f" 版本: {info.get('version', {}).get('number', 'N/A')}") + + # 尝试获取索引列表 + indices_response = requests.get( + f"{url}/_cat/indices?format=json", + auth=HTTPBasicAuth(user, password), + verify=False, + timeout=10 + ) + if indices_response.status_code == 200: + indices = indices_response.json() + print(f" 索引数量: {len(indices)}") + if indices: + print(f" 索引示例: {', '.join([idx['index'] for idx in indices[:3]])}") + + return True + else: + print(f"❌ 连接失败: HTTP {response.status_code}") + print(f" 响应: {response.text[:200]}") + return False + + except ImportError: + print(f"\n⚠️ 缺少 requests 库，无法测试 Elasticsearch") + print(f" 请运行: pip install requests") + return None + except Exception as e: + print(f"❌ 连接异常: {str(e)[:200]}") + return False + +def test_mysql_connection(host, port, user, password, description, database=None): + """测试 MySQL 连接""" + try: + import pymysql + + print(f"\n{'='*60}") + print(f"测试: {description}") + print(f"地址: {host}:{port}") + print(f"{'='*60}") + + # 尝试连接 + connection = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, + connect_timeout=10, + read_timeout=10 + ) + + print(f"✅ 连接成功!") + + # 获取服务器信息 + with connection.cursor() as cursor: + cursor.execute("SELECT VERSION()") + version = cursor.fetchone() + print(f" 版本: {version[0] if version else 'N/A'}") + + # 获取数据库列表 + cursor.execute("SHOW DATABASES") + databases = cursor.fetchall() + print(f" 数据库数量: {len(databases)}") + if databases: + print(f" 数据库示例: {', '.join([db[0] for db in databases[:5]])}") + + connection.close() + return True + + except ImportError: + print(f"\n⚠️ 缺少 pymysql 库，无法测试 MySQL") + print(f" 请运行: pip install pymysql") + return None + except Exception as e: + print(f"❌ 连接异常: {str(e)[:200]}") + return False + +def test_postgresql_connection(host, port, user, password, description, database=None): + """测试 PostgreSQL 连接""" + try: + import psycopg2 + + print(f"\n{'='*60}") + print(f"测试: {description}") + print(f"地址: {host}:{port}") + print(f"{'='*60}") + + # 尝试连接 + connection = psycopg2.connect( + host=host, + port=port, + user=user, + password=password, + dbname=database if database else 'postgres', + connect_timeout=10 + ) + + print(f"✅ 连接成功!") + + # 获取服务器信息 + with connection.cursor() as cursor: + cursor.execute("SELECT version()") + version = cursor.fetchone() + print(f" 版本: {version[0].split()[0] if version else 'N/A'}") + + # 获取数据库列表 + cursor.execute("SELECT datname FROM pg_database WHERE datistemplate = false") + databases = cursor.fetchall() + print(f" 数据库数量: {len(databases)}") + if databases: + print(f" 数据库示例: {', '.join([db[0] for db in databases[:5]])}") + + connection.close() + return True + + except ImportError: + print(f"\n⚠️ 缺少 psycopg2-binary 库，无法测试 PostgreSQL") + print(f" 请运行: pip install psycopg2-binary") + return None + except Exception as e: + print(f"❌ 连接异常: {str(e)[:200]}") + return False + +def main(): + print("="*60) + print("数据库连接测试") + print("注意: 仅进行连接测试和只读操作") + print("="*60) + + results = {} + + # ES 配置 + es_configs = [ + { + "description": "Test ES (测试环境服务日志)", + "host": "es-o79jsx9i.public.tencentelasticsearch.com", + "port": 9200, + "scheme": "https", + "user": "elastic", + "password": "lPLYr2!ap%^4UQb#" + }, + { + "description": "Online ES (正式环境服务日志)", + "host": "es-7vd7jcu9.public.tencentelasticsearch.com", + "port": 9200, + "scheme": "https", + "user": "elastic", + "password": "F%?QDcWes7N2WTuiYD11" + } + ] + + # MySQL 配置 + mysql_configs = [ + { + "description": "Online MySQL (线上版本)", + "host": "bj-cdb-dh2fkqa0.sql.tencentcdb.com", + "port": 27751, + "user": "read_only", + "password": "fsdo45ijfmfmuu77$%^&" + }, + { + "description": "Test MySQL (测试环境)", + "host": "bj-cdb-8frbdwju.sql.tencentcdb.com", + "port": 25413, + "user": "read_only", + "password": "fdsfiidier^$*hjfdijjd232" + } + ] + + # PostgreSQL 配置 + pg_configs = [ + { + "description": "Online PostgreSQL 1 (线上用户行为数据)", + "host": "bj-postgres-16pob4sg.sql.tencentcdb.com", + "port": 28591, + "user": "ai_member", + "password": "Jhfdhsfduse&%$*^&6786" + }, + { + "description": "Online PostgreSQL 2 (正式环境用户行为数据)", + "host": "bj-postgres-642mcico.sql.tencentcdb.com", + "port": 21531, + "user": "ai_member", + "password": "LdfjdjL83h3h3^$&**YGG*" + } + ] + + # 安装必要的库 + print("\n正在安装必要的 Python 库...") + import subprocess + try: + subprocess.check_call([sys.executable, "-m", "pip", "install", "--break-system-packages", "pymysql", "psycopg2-binary"]) + print("✅ 库安装成功!") + except Exception as e: + print(f"⚠️ 库安装可能遇到问题: {e}") + print(" 继续尝试测试...") + + # 测试 ES 连接 + print("\n" + "="*60) + print("测试 Elasticsearch 数据库") + print("="*60) + for config in es_configs: + result = test_es_connection(**config) + results[config["description"]] = result + + # 测试 MySQL 连接 + print("\n" + "="*60) + print("测试 MySQL 数据库") + print("="*60) + for config in mysql_configs: + result = test_mysql_connection(**config) + results[config["description"]] = result + + # 测试 PostgreSQL 连接 + print("\n" + "="*60) + print("测试 PostgreSQL 数据库") + print("="*60) + for config in pg_configs: + result = test_postgresql_connection(**config) + results[config["description"]] = result + + # 总结 + print("\n" + "="*60) + print("测试总结") + print("="*60) + for name, result in results.items(): + status = "✅ 成功" if result else ("❌ 失败" if result is False else "⚠️ 跳过") + print(f"{name}: {status}") + + print("\n📋 备注:") + print(" - Test PostgreSQL 配置缺少 host 和 port 信息") + print(" - 所有测试仅进行只读操作，未修改任何数据") + +if __name__ == "__main__": + main() diff --git a/test_mysql_pg.py b/test_mysql_pg.py new file mode 100644 index 0000000..7f31701 --- /dev/null +++ b/test_mysql_pg.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +""" +MySQL 和 PostgreSQL 连接测试脚本 +仅用于测试连接和读取基本信息，不进行任何写入操作 +""" + +import warnings +warnings.filterwarnings('ignore') + +def test_mysql_connection(host, port, user, password, description): + """测试 MySQL 连接""" + try: + import pymysql + + print(f"\n{'='*60}") + print(f"测试: {description}") + print(f"地址: {host}:{port}") + print(f"{'='*60}") + + # 尝试连接 + connection = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + connect_timeout=10, + read_timeout=10 + ) + + print(f"✅ 连接成功!") + + # 获取服务器信息 + with connection.cursor() as cursor: + cursor.execute("SELECT VERSION()") + version = cursor.fetchone() + print(f" 版本: {version[0] if version else 'N/A'}") + + # 获取数据库列表 + cursor.execute("SHOW DATABASES") + databases = cursor.fetchall() + print(f" 数据库数量: {len(databases)}") + if databases: + print(f" 数据库示例: {', '.join([db[0] for db in databases[:5]])}") + + connection.close() + return True + + except Exception as e: + print(f"❌ 连接异常: {str(e)[:200]}") + return False + +def test_postgresql_connection(host, port, user, password, description): + """测试 PostgreSQL 连接""" + try: + import psycopg2 + + print(f"\n{'='*60}") + print(f"测试: {description}") + print(f"地址: {host}:{port}") + print(f"{'='*60}") + + # 尝试连接 - 先尝试连接 postgres 数据库 + try: + connection = psycopg2.connect( + host=host, + port=port, + user=user, + password=password, + dbname='postgres', + connect_timeout=10 + ) + except: + # 如果 postgres 数据库连接失败，尝试不指定数据库 + print(f" 尝试不指定数据库连接...") + connection = psycopg2.connect( + host=host, + port=port, + user=user, + password=password, + connect_timeout=10 + ) + + print(f"✅ 连接成功!") + + # 获取服务器信息 + with connection.cursor() as cursor: + cursor.execute("SELECT version()") + version = cursor.fetchone() + print(f" 版本: {version[0].split()[0] if version else 'N/A'}") + + # 获取数据库列表 + try: + cursor.execute("SELECT datname FROM pg_database WHERE datistemplate = false") + databases = cursor.fetchall() + print(f" 数据库数量: {len(databases)}") + if databases: + print(f" 数据库示例: {', '.join([db[0] for db in databases[:5]])}") + except: + print(f" 无法获取数据库列表（权限限制）") + + connection.close() + return True + + except Exception as e: + print(f"❌ 连接异常: {str(e)[:200]}") + return False + +def main(): + print("="*60) + print("MySQL 和 PostgreSQL 数据库连接测试") + print("注意: 仅进行连接测试和只读操作") + print("="*60) + + results = {} + + # MySQL 配置 + mysql_configs = [ + { + "description": "Online MySQL (线上版本)", + "host": "bj-cdb-dh2fkqa0.sql.tencentcdb.com", + "port": 27751, + "user": "read_only", + "password": "fsdo45ijfmfmuu77$%^&" + }, + { + "description": "Test MySQL (测试环境)", + "host": "bj-cdb-8frbdwju.sql.tencentcdb.com", + "port": 25413, + "user": "read_only", + "password": "fdsfiidier^$*hjfdijjd232" + } + ] + + # PostgreSQL 配置（更新后的配置） + pg_configs = [ + { + "description": "Online PostgreSQL (正式环境用户行为数据)", + "host": "bj-postgres-16pob4sg.sql.tencentcdb.com", + "port": 28591, + "user": "ai_member", + "password": "LdfjdjL83h3h3^$&**YGG*" + }, + { + "description": "Test PostgreSQL (测试环境行为数据)", + "host": "bj-postgres-642mcico.sql.tencentcdb.com", + "port": 21531, + "user": "ai_member", + "password": "dsjsLGU&%$%FG*((yy9y8" + } + ] + + # 测试 MySQL 连接 + print("\n" + "="*60) + print("测试 MySQL 数据库") + print("="*60) + for config in mysql_configs: + result = test_mysql_connection(**config) + results[config["description"]] = result + + # 测试 PostgreSQL 连接 + print("\n" + "="*60) + print("测试 PostgreSQL 数据库") + print("="*60) + for config in pg_configs: + result = test_postgresql_connection(**config) + results[config["description"]] = result + + # 总结 + print("\n" + "="*60) + print("测试总结") + print("="*60) + for name, result in results.items(): + status = "✅ 成功" if result else "❌ 失败" + print(f"{name}: {status}") + +if __name__ == "__main__": + main() diff --git a/venv/bin/python b/venv/bin/python new file mode 120000 index 0000000..b8a0adb --- /dev/null +++ b/venv/bin/python @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/venv/bin/python3 b/venv/bin/python3 new file mode 120000 index 0000000..ae65fda --- /dev/null +++ b/venv/bin/python3 @@ -0,0 +1 @@ +/usr/bin/python3 \ No newline at end of file diff --git a/venv/bin/python3.12 b/venv/bin/python3.12 new file mode 120000 index 0000000..b8a0adb --- /dev/null +++ b/venv/bin/python3.12 @@ -0,0 +1 @@ +python3 \ No newline at end of file diff --git a/venv/lib64 b/venv/lib64 new file mode 120000 index 0000000..7951405 --- /dev/null +++ b/venv/lib64 @@ -0,0 +1 @@ +lib \ No newline at end of file diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg new file mode 100644 index 0000000..0d52065 --- /dev/null +++ b/venv/pyvenv.cfg @@ -0,0 +1,5 @@ +home = /usr/bin +include-system-site-packages = false +version = 3.12.3 +executable = /usr/bin/python3.12 +command = /usr/bin/python3 -m venv /root/.openclaw/workspace/venv