Implement HF Dataset for persistent device storage

- Add HFDatasetManager for HF dataset operations
- Replace file-based storage with HF dataset
- Add automatic dataset creation and management
- Implement fallback to in-memory storage
- Update all device save/load operations
- Add comprehensive error handling
- Include setup documentation

HF-DATASET-SETUP.md

@@ -0,0 +1,169 @@
+# 🤗 Hugging Face Dataset Setup Guide
+
+## Overview
+
+This server now uses **Hugging Face Dataset** for persistent storage of device tokens instead of local files. This ensures that device data is preserved across Space restarts and provides better data management.
+
+## 🚀 Quick Setup
+
+### 1. Generate Hugging Face Token
+
+1. Go to [Hugging Face Settings](https://huggingface.co/settings/tokens)
+2. Click "Create new token"
+3. Name: `houzou-notification-server`
+4. Type: **Write** (required for dataset operations)
+5. Copy the token
+
+### 2. Configure Space Secrets
+
+1. Go to your Space Settings → **Secrets and variables**
+2. Add these secrets:
+
+```
+HF_TOKEN = hf_your_token_here
+HF_DATASET_ID = Detomo/houzou-devices  (optional, defaults to this)
+```
+
+### 3. Required Secrets Summary
+
+Your Space needs these secrets:
+
+```
+FIREBASE_SERVICE_ACCOUNT = {your firebase config JSON}
+HF_TOKEN = hf_your_write_token_here
+HF_DATASET_ID = Detomo/houzou-devices (optional)
+```
+
+## 📊 Dataset Structure
+
+The dataset will be automatically created with:
+
+```
+datasets/Detomo/houzou-devices/
+├── README.md           # Dataset documentation
+└── devices.json        # Device tokens and metadata
+```
+
+### devices.json Format:
+```json
+[
+  [
+    "device-id-1",
+    {
+      "token": "fcm_token_here",
+      "lastUpdated": "2024-01-01T00:00:00.000Z",
+      "platform": "iOS",
+      "appVersion": "1.0.0",
+      "registered": true
+    }
+  ]
+]
+```
+
+## 🔧 Features
+
+### ✅ Automatic Dataset Management
+- Creates dataset if it doesn't exist
+- Handles permissions automatically
+- Provides fallback to in-memory storage
+
+### ✅ Persistent Storage
+- Device tokens survive Space restarts
+- Automatic backup to HF dataset
+- Version control for device data
+
+### ✅ Error Handling
+- Graceful fallback if HF API is unavailable
+- Detailed logging for troubleshooting
+- Continues operation even if dataset access fails
+
+## 🔍 Verification
+
+### Check Dataset Status
+```bash
+curl https://your-space-url.hf.space/
+```
+
+Response includes:
+```json
+{
+  "message": "Houzou Medical Notification Server",
+  "status": "running",
+  "hfDataset": {
+    "enabled": true,
+    "datasetId": "Detomo/houzou-devices",
+    "hasToken": true
+  },
+  "deviceCount": 5
+}
+```
+
+### View Dataset
+1. Go to: https://huggingface.co/datasets/Detomo/houzou-devices
+2. Check `devices.json` for your device data
+
+## 🐛 Troubleshooting
+
+### Dataset Not Created
+- Verify `HF_TOKEN` has **Write** permissions
+- Check Space logs for error messages
+- Ensure token is valid and not expired
+
+### Data Not Saving
+- Check if HF API is responding
+- Verify dataset permissions
+- Server will continue with in-memory storage as fallback
+
+### Common Errors
+
+**403 Forbidden**
+- Token doesn't have write permissions
+- Token is expired or invalid
+
+**404 Not Found**
+- Dataset doesn't exist (will be auto-created)
+- Wrong dataset ID
+
+**Rate Limiting**
+- HF API has rate limits
+- Server will retry automatically
+
+## 🔄 Migration from File Storage
+
+If you're migrating from file-based storage:
+
+1. **Backup existing data**: Download your current `devices.json`
+2. **Set up HF dataset**: Follow steps above
+3. **Deploy updated server**: The server will automatically use HF dataset
+4. **Verify**: Check that devices are loaded correctly
+
+## 📋 Environment Variables
+
+```bash
+# Required
+HF_TOKEN=hf_your_write_token_here
+FIREBASE_SERVICE_ACCOUNT={"type":"service_account",...}
+
+# Optional
+HF_DATASET_ID=Detomo/houzou-devices
+PORT=7860
+NODE_ENV=production
+```
+
+## 🎯 Benefits
+
+- **✅ Persistent**: Data survives Space restarts
+- **✅ Reliable**: Automatic fallback mechanisms
+- **✅ Scalable**: No file permission issues
+- **✅ Traceable**: Version history in dataset
+- **✅ Secure**: Tokens stored in HF infrastructure
+
+## 🔗 Links
+
+- [Hugging Face Hub API](https://huggingface.co/docs/hub/api)
+- [Dataset Management](https://huggingface.co/docs/datasets/)
+- [Access Tokens](https://huggingface.co/settings/tokens)
+
+---
+
+**Ready to deploy!** Your notification server will now use HF Dataset for persistent storage. 🚀 

hf-dataset.js

@@ -0,0 +1,177 @@
+const { HfApi } = require("@huggingface/hub");
+const fetch = require('node-fetch');
+
+class HFDatasetManager {
+  constructor() {
+    this.hfToken = process.env.HF_TOKEN;
+    this.datasetId = process.env.HF_DATASET_ID || "Detomo/houzou-devices";
+    this.fileName = "devices.json";
+    this.hfApi = new HfApi({ accessToken: this.hfToken });
+    this.isEnabled = !!this.hfToken;
+    
+    console.log(`🤗 HF Dataset Manager initialized`);
+    console.log(`   Dataset: ${this.datasetId}`);
+    console.log(`   Enabled: ${this.isEnabled}`);
+  }
+
+  async loadDevices() {
+    if (!this.isEnabled) {
+      console.log('⚠️ HF Dataset disabled - no token provided');
+      return new Map();
+    }
+
+    try {
+      console.log('📥 Loading devices from HF dataset...');
+      
+      const fileUrl = `https://huggingface.co/datasets/${this.datasetId}/resolve/main/${this.fileName}`;
+      const response = await fetch(fileUrl, {
+        headers: {
+          'Authorization': `Bearer ${this.hfToken}`
+        }
+      });
+
+      if (response.status === 404) {
+        console.log('📁 No devices file found in dataset, creating new one');
+        await this.saveDevices(new Map());
+        return new Map();
+      }
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+      }
+
+      const data = await response.json();
+      const deviceMap = new Map(data);
+      
+      console.log(`📥 Loaded ${deviceMap.size} devices from HF dataset`);
+      return deviceMap;
+      
+    } catch (error) {
+      console.error('❌ Error loading devices from HF dataset:', error);
+      console.log('⚠️ Falling back to empty device list');
+      return new Map();
+    }
+  }
+
+  async saveDevices(deviceMap) {
+    if (!this.isEnabled) {
+      console.log('⚠️ HF Dataset disabled - cannot save devices');
+      return false;
+    }
+
+    try {
+      console.log(`💾 Saving ${deviceMap.size} devices to HF dataset...`);
+      
+      const devicesArray = Array.from(deviceMap.entries());
+      const jsonData = JSON.stringify(devicesArray, null, 2);
+      
+      // Create a blob from the JSON data
+      const blob = new Blob([jsonData], { type: 'application/json' });
+      
+      // Upload to HF dataset
+      await this.hfApi.uploadFile({
+        repo: this.datasetId,
+        file: {
+          path: this.fileName,
+          content: blob
+        },
+        commitMessage: `Update devices data - ${new Date().toISOString()}`,
+        repoType: "dataset"
+      });
+      
+      console.log(`✅ Successfully saved ${deviceMap.size} devices to HF dataset`);
+      return true;
+      
+    } catch (error) {
+      console.error('❌ Error saving devices to HF dataset:', error);
+      return false;
+    }
+  }
+
+  async createDatasetIfNotExists() {
+    if (!this.isEnabled) {
+      console.log('⚠️ HF Dataset disabled - cannot create dataset');
+      return false;
+    }
+
+    try {
+      console.log('🔍 Checking if dataset exists...');
+      
+      // Try to get dataset info
+      const datasetInfo = await this.hfApi.datasetInfo({
+        repo: this.datasetId
+      });
+      
+      console.log('✅ Dataset already exists');
+      return true;
+      
+    } catch (error) {
+      if (error.statusCode === 404) {
+        console.log('📁 Dataset not found, creating new one...');
+        
+        try {
+          // Create the dataset
+          await this.hfApi.createRepo({
+            repo: this.datasetId,
+            type: "dataset",
+            private: false
+          });
+          
+          console.log('✅ Dataset created successfully');
+          
+          // Create initial README
+          const readmeContent = `# Houzou Medical Devices Dataset
+
+This dataset stores FCM tokens and device information for the Houzou Medical app notification system.
+
+## Files
+
+- \`devices.json\`: Contains device tokens and metadata
+
+## Usage
+
+This dataset is automatically managed by the Houzou Medical Notification Server.
+
+Last updated: ${new Date().toISOString()}
+`;
+          
+          await this.hfApi.uploadFile({
+            repo: this.datasetId,
+            file: {
+              path: "README.md",
+              content: new Blob([readmeContent], { type: 'text/markdown' })
+            },
+            commitMessage: "Initial dataset setup",
+            repoType: "dataset"
+          });
+          
+          // Create initial empty devices file
+          await this.saveDevices(new Map());
+          
+          return true;
+          
+        } catch (createError) {
+          console.error('❌ Error creating dataset:', createError);
+          return false;
+        }
+      } else {
+        console.error('❌ Error checking dataset:', error);
+        return false;
+      }
+    }
+  }
+
+  isReady() {
+    return this.isEnabled;
+  }
+
+  getStatus() {
+    return {
+      enabled: this.isEnabled,
+      datasetId: this.datasetId,
+      hasToken: !!this.hfToken
+    };
+  }
+}
+
+module.exports = HFDatasetManager; 

package.json

@@ -12,7 +12,9 @@
     "firebase-admin": "^12.0.0",
     "cors": "^2.8.5",
     "body-parser": "^1.20.2",
-    "dotenv": "^16.3.1"
+    "dotenv": "^16.3.1",
+    "@huggingface/hub": "^0.15.1",
+    "node-fetch": "^3.3.2"
   },
   "devDependencies": {
     "nodemon": "^3.0.2"

server.js

@@ -3,6 +3,7 @@ const admin = require('firebase-admin');
 const cors = require('cors');
 const bodyParser = require('body-parser');
 const os = require('os');
+const HFDatasetManager = require('./hf-dataset');
 require('dotenv').config();
 
 const app = express();
@@ -59,60 +60,52 @@ const sampleProducts = [
 const fs = require('fs');
 const path = require('path');
 
-// File storage for FCM tokens
-const DEVICES_FILE = path.join(__dirname, 'devices.json');
+// Initialize HF Dataset Manager
+const hfDataset = new HFDatasetManager();
 
-// Fallback to in-memory storage if file operations fail
-let fileOperationsEnabled = true;
-
-// Load devices from file or create empty storage
+// Device storage
 let deviceTokens = new Map();
 
-function loadDevicesFromFile() {
+async function loadDevicesFromHF() {
   try {
-    if (fs.existsSync(DEVICES_FILE)) {
-      const data = fs.readFileSync(DEVICES_FILE, 'utf8');
-      const devicesArray = JSON.parse(data);
-      deviceTokens = new Map(devicesArray);
-      console.log(`📁 Loaded ${deviceTokens.size} devices from file`);
-    } else {
-      console.log('📁 No devices file found, starting fresh');
-      // Try to create an empty file to test write permissions
-      try {
-        fs.writeFileSync(DEVICES_FILE, '[]');
-        console.log('📁 Created empty devices file');
-      } catch (writeError) {
-        console.warn('⚠️ Cannot create devices file, will use in-memory storage only');
-        fileOperationsEnabled = false;
-      }
-    }
+    deviceTokens = await hfDataset.loadDevices();
+    console.log(`📥 Loaded ${deviceTokens.size} devices from HF dataset`);
   } catch (error) {
-    console.error('❌ Error loading devices file:', error);
-    console.log('⚠️ Using in-memory storage only');
-    fileOperationsEnabled = false;
+    console.error('❌ Error loading devices from HF dataset:', error);
     deviceTokens = new Map();
   }
 }
 
-function saveDevicesToFile() {
-  if (!fileOperationsEnabled) {
-    console.log('💾 File operations disabled, keeping devices in memory only');
-    return;
+async function saveDevicesToHF() {
+  try {
+    const success = await hfDataset.saveDevices(deviceTokens);
+    if (success) {
+      console.log(`💾 Saved ${deviceTokens.size} devices to HF dataset`);
+    } else {
+      console.log('⚠️ Failed to save devices to HF dataset');
+    }
+  } catch (error) {
+    console.error('❌ Error saving devices to HF dataset:', error);
   }
-  
+}
+
+// Initialize HF dataset and load devices on startup
+async function initializeServer() {
   try {
-    const devicesArray = Array.from(deviceTokens.entries());
-    fs.writeFileSync(DEVICES_FILE, JSON.stringify(devicesArray, null, 2));
-    console.log(`💾 Saved ${deviceTokens.size} devices to file`);
+    // Create dataset if it doesn't exist
+    await hfDataset.createDatasetIfNotExists();
+    
+    // Load devices from HF dataset
+    await loadDevicesFromHF();
+    
+    console.log('✅ Server initialization completed');
   } catch (error) {
-    console.error('❌ Error saving devices file:', error);
-    console.log('⚠️ Disabling file operations, using in-memory storage only');
-    fileOperationsEnabled = false;
+    console.error('❌ Error during server initialization:', error);
   }
 }
 
-// Load devices on startup
-loadDevicesFromFile();
+// Start initialization (don't await here to avoid blocking)
+initializeServer();
 
 // Routes
 
@@ -122,7 +115,7 @@ app.get('/', (req, res) => {
     message: 'Houzou Medical Notification Server',
     status: 'running',
     timestamp: new Date().toISOString(),
-    fileOperationsEnabled: fileOperationsEnabled,
+    hfDataset: hfDataset.getStatus(),
     deviceCount: deviceTokens.size
   });
 });
@@ -433,8 +426,8 @@ app.post('/register-token', async (req, res) => {
       console.warn(`⚠️ Failed to subscribe device to topic: ${topicError.message}`);
     }
 
-    // Save to file
-    saveDevicesToFile();
+    // Save to HF dataset
+    saveDevicesToHF();
 
     res.json({
       success: true,
@@ -476,8 +469,8 @@ app.post('/unregister-token', async (req, res) => {
       console.log(`📱 Token unregistered for device ${deviceId}`);
     }
 
-    // Save to file
-    saveDevicesToFile();
+    // Save to HF dataset
+    saveDevicesToHF();
 
     res.json({
       success: true,
@@ -531,7 +524,7 @@ app.post('/debug-add-device', (req, res) => {
       registered: true
     });
     
-    saveDevicesToFile();
+    saveDevicesToHF();
     
     console.log(`🔧 Debug: Added device ${deviceId} (${platform})`);
     
@@ -642,7 +635,7 @@ app.post('/send-broadcast-notification', async (req, res) => {
             }
           }
         });
-        saveDevicesToFile();
+        saveDevicesToHF();
       } else {
         console.log(`⚠️ No tokens removed - all failures appear to be temporary`);
       }
@@ -701,7 +694,7 @@ app.listen(PORT, () => {
   console.log(`   Local:    http://localhost:${PORT}`);
   console.log(`   Network:  http://${localIP}:${PORT}`);
   console.log(`\n🔧 For iPhone app, use: http://${localIP}:${PORT}`);
-  console.log(`📁 Devices saved to: ${DEVICES_FILE}`);
+  console.log(`🤗 Devices stored in HF dataset: ${hfDataset.datasetId}`);
 });
 
 module.exports = app;