linxinhua commited on
Commit
6ce256d
·
verified ·
1 Parent(s): e23938f

Update app.py

Browse files

download file name

Files changed (1) hide show
  1. app.py +59 -57
app.py CHANGED
@@ -13,28 +13,28 @@ class DataManager:
13
  self.repo_id = "CIV3283/Data_Storage"
14
  self.branch = "data_branch"
15
 
16
- # 优先使用传入的token,否则从环境变量获取
17
  self.token = token or os.getenv('CIV3283_access')
18
 
19
- # 如果有token,进行认证
20
  if self.token:
21
  try:
22
  login(token=self.token)
23
- print("✅ 认证成功")
24
  except Exception as e:
25
- print(f"❌ 认证失败: {e}")
26
 
27
  self.api = HfApi(token=self.token)
28
 
29
  def get_available_files(self) -> Tuple[List[str], List[str]]:
30
- """获取所有可用的feedback和query文件"""
31
  try:
32
- # 尝试获取仓库中的所有文件
33
  repo_files = self.api.list_repo_files(
34
  repo_id=self.repo_id,
35
  revision=self.branch,
36
  token=self.token,
37
- repo_type="space" # 明确指定这是一个space
38
  )
39
 
40
  feedback_files = []
@@ -49,18 +49,18 @@ class DataManager:
49
  return sorted(feedback_files), sorted(query_files)
50
 
51
  except Exception as e:
52
- print(f"❌ 获取文件列表错误: {e}")
53
- # 如果API调用失败,尝试手动生成可能的文件名列表
54
  return self._generate_possible_filenames()
55
 
56
  def _generate_possible_filenames(self) -> Tuple[List[str], List[str]]:
57
- """生成可能的文件名列表作为后备方案"""
58
  feedback_files = []
59
  query_files = []
60
 
61
- # 生成01到95的所有可能文件名
62
  for i in range(1, 96):
63
- student_id = f"{i:02d}" # 格式化为两位数
64
  feedback_file = f"CIV3283_Student_{student_id}_feedback_log.csv"
65
  query_file = f"CIV3283_Student_{student_id}_query_log.csv"
66
  feedback_files.append(feedback_file)
@@ -69,9 +69,9 @@ class DataManager:
69
  return feedback_files, query_files
70
 
71
  def download_and_read_csv(self, filename: str) -> pd.DataFrame:
72
- """下载并读取CSV文件"""
73
  try:
74
- # 下载文件到临时位置
75
  local_file = hf_hub_download(
76
  repo_id=self.repo_id,
77
  filename=filename,
@@ -81,35 +81,35 @@ class DataManager:
81
  cache_dir=tempfile.gettempdir()
82
  )
83
 
84
- # 读取CSV
85
  df = pd.read_csv(local_file)
86
 
87
- # 确保timestamp列存在并转换为datetime
88
  if 'timestamp' in df.columns:
89
  df['timestamp'] = pd.to_datetime(df['timestamp'])
90
 
91
  return df
92
 
93
  except Exception as e:
94
- print(f"❌ 读取文件 {filename} 失败: {e}")
95
- # 返回空DataFrame而不是完全失败
96
  return pd.DataFrame()
97
 
98
  def filter_data_by_date(self, df: pd.DataFrame, start_date: str, end_date: str) -> pd.DataFrame:
99
- """根据日期范围筛选数据"""
100
  if df.empty or 'timestamp' not in df.columns:
101
  return df
102
 
103
  try:
104
- # 转换输入的日期字符串
105
  start_dt = pd.to_datetime(start_date)
106
 
107
- if end_date == "现在":
108
  end_dt = pd.Timestamp.now()
109
  else:
110
- end_dt = pd.to_datetime(end_date) + timedelta(days=1) - timedelta(seconds=1) # 包含整天
111
 
112
- # 筛选数据
113
  mask = (df['timestamp'] >= start_dt) & (df['timestamp'] <= end_dt)
114
  return df[mask]
115
 
@@ -118,21 +118,21 @@ class DataManager:
118
  return df
119
 
120
  def process_files(self, file_type: str, start_date: str, end_date: str) -> Tuple[str, str, str]:
121
- """处理文件并生成下载内容"""
122
  feedback_files, query_files = self.get_available_files()
123
 
124
  result_data = []
125
  file_info = []
126
  total_records = 0
127
 
128
- # 根据选择的文件类型确定要处理的文件
129
  files_to_process = []
130
- if file_type in ["反馈文件", "两者都要"]:
131
  files_to_process.extend([(f, "feedback") for f in feedback_files])
132
- if file_type in ["查询文件", "两者都要"]:
133
  files_to_process.extend([(f, "query") for f in query_files])
134
 
135
- # 处理每个文件
136
  for filename, ftype in files_to_process:
137
  df = self.download_and_read_csv(filename)
138
  if not df.empty:
@@ -144,44 +144,45 @@ class DataManager:
144
  result_data.append(filtered_df)
145
  total_records += filtered_count
146
 
147
- file_info.append(f"{filename}: {original_count}条记录 → {filtered_count}条符合条件")
148
 
149
- # 合并所有数据
150
  if result_data:
151
  combined_df = pd.concat(result_data, ignore_index=True)
152
 
153
- # 生成CSV内容
154
  csv_buffer = io.StringIO()
155
  combined_df.to_csv(csv_buffer, index=False, encoding='utf-8')
156
  csv_content = csv_buffer.getvalue()
157
 
158
- # 生成文件名
159
- end_date_str = "现在" if end_date == "现在" else end_date
160
- filename = f"CIV3283_数据导出_{file_type}_{start_date}_to_{end_date_str}.csv"
 
161
 
162
- # 生成状态信息
163
- status = f"✅ 成功处理 {len(files_to_process)} 个文件\n"
164
- status += f"📊 总共找到 {total_records} 条符合条件的记录\n"
165
- status += f"📅 时间范围: {start_date} 到 {end_date_str}\n\n"
166
- status += "文件详情:\n" + "\n".join(file_info)
167
 
168
  return csv_content, filename, status
169
  else:
170
- return "", "", "❌ 没有找到符合条件的数据"
171
 
172
- # 全局变量存储数据管理器
173
  data_manager = None
174
 
175
  def initialize_data_manager(hf_token=None):
176
- """初始化数据管理器"""
177
  global data_manager
178
  data_manager = DataManager(token=hf_token)
179
  return data_manager
180
 
181
  def process_and_download(file_type, start_date, end_date, hf_token):
182
- """处理请求并返回下载文件"""
183
  try:
184
- # 确保数据管理器已初始化
185
  global data_manager
186
  env_token = os.getenv('CIV3283_access')
187
  current_token = hf_token or env_token
@@ -192,7 +193,7 @@ def process_and_download(file_type, start_date, end_date, hf_token):
192
  csv_content, filename, status = data_manager.process_files(file_type, start_date, end_date)
193
 
194
  if csv_content:
195
- # 创建临时文件
196
  temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False, encoding='utf-8')
197
  temp_file.write(csv_content)
198
  temp_file.close()
@@ -202,12 +203,12 @@ def process_and_download(file_type, start_date, end_date, hf_token):
202
  return status, None, None
203
 
204
  except Exception as e:
205
- return f"❌ 处理过程中出现错误: {str(e)}", None, None
206
 
207
  def get_file_summary(hf_token=None):
208
- """获取文件概览"""
209
  try:
210
- # 如果还没有初始化或token发生变化,重新初始化
211
  global data_manager
212
  env_token = os.getenv('CIV3283_access')
213
  current_token = hf_token or env_token
@@ -216,21 +217,21 @@ def get_file_summary(hf_token=None):
216
  data_manager = initialize_data_manager(current_token)
217
 
218
  feedback_files, query_files = data_manager.get_available_files()
219
- summary = f"📁 当前可用文件:\n"
220
- summary += f" 反馈文件: {len(feedback_files)} 个\n"
221
- summary += f" 查询文件: {len(query_files)} 个\n"
222
- summary += f" 总计: {len(feedback_files) + len(query_files)} 个文件"
223
 
224
  if env_token:
225
- summary += f"\n✅ 使用环境变量 CIV3283_access 认证"
226
  elif hf_token:
227
- summary += f"\n✅ 使用手动输入的token认证"
228
  else:
229
- summary += f"\n⚠️ 未找到认证token(可能无法访问私有仓库)"
230
 
231
  return summary
232
  except Exception as e:
233
- return f"❌ 获取文件列表失败: {str(e)}"
234
 
235
  # 创建Gradio界面
236
  def create_interface():
@@ -352,8 +353,9 @@ def create_interface():
352
 
353
  return demo
354
 
355
- # 启动应用
356
  if __name__ == "__main__":
357
  demo = create_interface()
358
  demo.launch(
 
359
  )
 
13
  self.repo_id = "CIV3283/Data_Storage"
14
  self.branch = "data_branch"
15
 
16
+ # Use provided token first, otherwise get from environment variable
17
  self.token = token or os.getenv('CIV3283_access')
18
 
19
+ # Authenticate if token is available
20
  if self.token:
21
  try:
22
  login(token=self.token)
23
+ print("✅ Authentication successful")
24
  except Exception as e:
25
+ print(f"❌ Authentication failed: {e}")
26
 
27
  self.api = HfApi(token=self.token)
28
 
29
  def get_available_files(self) -> Tuple[List[str], List[str]]:
30
+ """Get all available feedback and query files"""
31
  try:
32
+ # Try to get all files from repository
33
  repo_files = self.api.list_repo_files(
34
  repo_id=self.repo_id,
35
  revision=self.branch,
36
  token=self.token,
37
+ repo_type="space" # Explicitly specify this is a space
38
  )
39
 
40
  feedback_files = []
 
49
  return sorted(feedback_files), sorted(query_files)
50
 
51
  except Exception as e:
52
+ print(f"❌ Error getting file list: {e}")
53
+ # If API call fails, try to generate possible filename list
54
  return self._generate_possible_filenames()
55
 
56
  def _generate_possible_filenames(self) -> Tuple[List[str], List[str]]:
57
+ """Generate possible filename list as backup"""
58
  feedback_files = []
59
  query_files = []
60
 
61
+ # Generate all possible filenames from 01 to 95
62
  for i in range(1, 96):
63
+ student_id = f"{i:02d}" # Format as two digits
64
  feedback_file = f"CIV3283_Student_{student_id}_feedback_log.csv"
65
  query_file = f"CIV3283_Student_{student_id}_query_log.csv"
66
  feedback_files.append(feedback_file)
 
69
  return feedback_files, query_files
70
 
71
  def download_and_read_csv(self, filename: str) -> pd.DataFrame:
72
+ """Download and read CSV file"""
73
  try:
74
+ # Download file to temporary location
75
  local_file = hf_hub_download(
76
  repo_id=self.repo_id,
77
  filename=filename,
 
81
  cache_dir=tempfile.gettempdir()
82
  )
83
 
84
+ # Read CSV
85
  df = pd.read_csv(local_file)
86
 
87
+ # Ensure timestamp column exists and convert to datetime
88
  if 'timestamp' in df.columns:
89
  df['timestamp'] = pd.to_datetime(df['timestamp'])
90
 
91
  return df
92
 
93
  except Exception as e:
94
+ print(f"❌ Failed to read file {filename}: {e}")
95
+ # Return empty DataFrame rather than complete failure
96
  return pd.DataFrame()
97
 
98
  def filter_data_by_date(self, df: pd.DataFrame, start_date: str, end_date: str) -> pd.DataFrame:
99
+ """Filter data by date range"""
100
  if df.empty or 'timestamp' not in df.columns:
101
  return df
102
 
103
  try:
104
+ # Convert input date strings
105
  start_dt = pd.to_datetime(start_date)
106
 
107
+ if end_date == "now":
108
  end_dt = pd.Timestamp.now()
109
  else:
110
+ end_dt = pd.to_datetime(end_date) + timedelta(days=1) - timedelta(seconds=1) # Include full day
111
 
112
+ # Filter data
113
  mask = (df['timestamp'] >= start_dt) & (df['timestamp'] <= end_dt)
114
  return df[mask]
115
 
 
118
  return df
119
 
120
  def process_files(self, file_type: str, start_date: str, end_date: str) -> Tuple[str, str, str]:
121
+ """Process files and generate download content"""
122
  feedback_files, query_files = self.get_available_files()
123
 
124
  result_data = []
125
  file_info = []
126
  total_records = 0
127
 
128
+ # Determine files to process based on selected file type
129
  files_to_process = []
130
+ if file_type in ["Feedback Files", "Both"]:
131
  files_to_process.extend([(f, "feedback") for f in feedback_files])
132
+ if file_type in ["Query Files", "Both"]:
133
  files_to_process.extend([(f, "query") for f in query_files])
134
 
135
+ # Process each file
136
  for filename, ftype in files_to_process:
137
  df = self.download_and_read_csv(filename)
138
  if not df.empty:
 
144
  result_data.append(filtered_df)
145
  total_records += filtered_count
146
 
147
+ file_info.append(f"{filename}: {original_count} records → {filtered_count} matching criteria")
148
 
149
+ # Combine all data
150
  if result_data:
151
  combined_df = pd.concat(result_data, ignore_index=True)
152
 
153
+ # Generate CSV content
154
  csv_buffer = io.StringIO()
155
  combined_df.to_csv(csv_buffer, index=False, encoding='utf-8')
156
  csv_content = csv_buffer.getvalue()
157
 
158
+ # Generate filename with type and date info
159
+ end_date_str = "now" if end_date == "now" else end_date
160
+ type_str = file_type.lower().replace(" ", "_")
161
+ filename = f"CIV3283_data_export_{type_str}_{start_date}_to_{end_date_str}.csv"
162
 
163
+ # Generate status info
164
+ status = f"✅ Successfully processed {len(files_to_process)} files\n"
165
+ status += f"📊 Total records found: {total_records}\n"
166
+ status += f"📅 Date range: {start_date} to {end_date_str}\n\n"
167
+ status += "File details:\n" + "\n".join(file_info)
168
 
169
  return csv_content, filename, status
170
  else:
171
+ return "", "", "❌ No matching data found"
172
 
173
+ # Global variable to store data manager
174
  data_manager = None
175
 
176
  def initialize_data_manager(hf_token=None):
177
+ """Initialize data manager"""
178
  global data_manager
179
  data_manager = DataManager(token=hf_token)
180
  return data_manager
181
 
182
  def process_and_download(file_type, start_date, end_date, hf_token):
183
+ """Process request and return download file"""
184
  try:
185
+ # Ensure data manager is initialized
186
  global data_manager
187
  env_token = os.getenv('CIV3283_access')
188
  current_token = hf_token or env_token
 
193
  csv_content, filename, status = data_manager.process_files(file_type, start_date, end_date)
194
 
195
  if csv_content:
196
+ # Create temporary file
197
  temp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False, encoding='utf-8')
198
  temp_file.write(csv_content)
199
  temp_file.close()
 
203
  return status, None, None
204
 
205
  except Exception as e:
206
+ return f"❌ Error occurred during processing: {str(e)}", None, None
207
 
208
  def get_file_summary(hf_token=None):
209
+ """Get file overview"""
210
  try:
211
+ # If not initialized or token changed, reinitialize
212
  global data_manager
213
  env_token = os.getenv('CIV3283_access')
214
  current_token = hf_token or env_token
 
217
  data_manager = initialize_data_manager(current_token)
218
 
219
  feedback_files, query_files = data_manager.get_available_files()
220
+ summary = f"📁 Currently available files:\n"
221
+ summary += f" Feedback files: {len(feedback_files)}\n"
222
+ summary += f" Query files: {len(query_files)}\n"
223
+ summary += f" Total: {len(feedback_files) + len(query_files)} files"
224
 
225
  if env_token:
226
+ summary += f"\n✅ Authenticated using environment variable CIV3283_access"
227
  elif hf_token:
228
+ summary += f"\n✅ Authenticated using manual token input"
229
  else:
230
+ summary += f"\n⚠️ No authentication token found (may not access private repositories)"
231
 
232
  return summary
233
  except Exception as e:
234
+ return f"❌ Failed to get file list: {str(e)}"
235
 
236
  # 创建Gradio界面
237
  def create_interface():
 
353
 
354
  return demo
355
 
356
+ # Launch application
357
  if __name__ == "__main__":
358
  demo = create_interface()
359
  demo.launch(
360
+
361
  )