diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..ff785508542e0fc712ab89d451edfac83de8dd5e --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +AWS_SECRET_ACCESS_KEY=+cGEaSdDgvg/NRNtwVdMarK5RYHMqEwxjvgcOnpD +AWS_ACCESS_KEY_ID=AKIA4AABJOVKM2P5RNXJ +AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1 +AWS_S3_URL=https://lectus-bucket.s3.us-east-1.amazonaws.com/ diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000000000000000000000000000000000000..37224185490e6db2d26a574d66d4d476336bf644 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": ["next/core-web-vitals", "next/typescript"] +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d32cc78b89fc9af2b1caf304864e10f041df05e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000000000000000000000000000000000000..06f005e3917482ad4af7283c6807899e0cfd8dcb --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,14 @@ +# This configuration file was automatically generated by Gitpod. +# Please adjust to your needs (see https://www.gitpod.io/docs/introduction/learn-gitpod/gitpod-yaml) +# and commit this file to your remote git repository to share the goodness with others. + +# Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart + +tasks: + - init: pnpm install && pnpm run build + command: pnpm run start + +ports: + - port: 3000 + onOpen: open-browser + visibility: public diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..13566b81b018ad684f3a35fee301741b2734c8f4 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000000000000000000000000000000000000..03d9549ea8e4ada36fb3ecbc30fef08175b7d728 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/jsLinters/eslint.xml b/.idea/jsLinters/eslint.xml new file mode 100644 index 0000000000000000000000000000000000000000..541945bb0819b8ff4a3dae9431632ebd10e6f98b --- /dev/null +++ b/.idea/jsLinters/eslint.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000000000000000000000000000000000000..37c53b85d40cd59dcea392c8286aeb628bedf870 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/prettier.xml b/.idea/prettier.xml new file mode 100644 index 0000000000000000000000000000000000000000..0c83ac4e895bb2cdb1aa51f61869e3bf7ccf4ec5 --- /dev/null +++ b/.idea/prettier.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000000000000000000000000000000000000..35eb1ddfbbc029bcab630581847471d7f238ec53 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/visio.iml b/.idea/visio.iml new file mode 100644 index 0000000000000000000000000000000000000000..24643cc37449b4bde54411a80b8ed61258225e34 --- /dev/null +++ b/.idea/visio.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000000000000000000000000000000000000..714c7dd41daf5552607fefea459c0251d5df6b7f --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,64 @@ + + + + + + + + + + { + "associatedIndex": 3 +} + + + + + + + + + + + + + + + 1732289755049 + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..3feaf9c7ec96829d2e197990309e8fc034fd488f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1.4 + +# Adapted from https://github.com/vercel/next.js/blob/e60a1e747c3f521fc24dfd9ee2989e13afeb0a9b/examples/with-docker/Dockerfile +# For more information, see https://nextjs.org/docs/pages/building-your-application/deploying#docker-image + +FROM node:18 AS base + +# Install dependencies only when needed +FROM base AS deps +WORKDIR /app + +# Install dependencies based on the preferred package manager +COPY --link package.json yarn.lock* package-lock.json* pnpm-lock.yaml* ./ +RUN \ + if [ -f yarn.lock ]; then yarn --frozen-lockfile; \ + elif [ -f package-lock.json ]; then npm ci; \ + elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \ + else echo "Lockfile not found." && exit 1; \ + fi + + +# Rebuild the source code only when needed +FROM base AS builder +WORKDIR /app +COPY --from=deps --link /app/node_modules ./node_modules +COPY --link . . + +# Next.js collects completely anonymous telemetry data about general usage. +# Learn more here: https://nextjs.org/telemetry +# Uncomment the following line in case you want to disable telemetry during the build. +# ENV NEXT_TELEMETRY_DISABLED 1 + +RUN npm run build + +# If using yarn comment out above and use below instead +# RUN yarn build + +# Production image, copy all the files and run next +FROM base AS runner +WORKDIR /app + +ENV NODE_ENV production +# Uncomment the following line in case you want to disable telemetry during runtime. +# ENV NEXT_TELEMETRY_DISABLED 1 + +RUN \ + addgroup --system --gid 1001 nodejs; \ + adduser --system --uid 1001 nextjs + +COPY --from=builder --link /app/public ./public + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=builder --link --chown=1001:1001 /app/.next/standalone ./ +COPY --from=builder --link --chown=1001:1001 /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 + +ENV PORT 3000 +ENV HOSTNAME 0.0.0.0 + +# Allow the running process to write model files to the cache folder. +# NOTE: In practice, you would probably want to pre-download the model files to avoid having to download them on-the-fly. +RUN mkdir -p /app/node_modules/@xenova/.cache/ +RUN chmod 777 -R /app/node_modules/@xenova/ + +CMD ["node", "server.js"] diff --git a/README.md b/README.md index e2f5f9c6e69a518a671dd9380f8d06ce04e52401..e215bc4ccf138bbc38ad58ad57e92135484b3c0f 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,36 @@ ---- -title: Visio -emoji: 😻 -colorFrom: pink -colorTo: gray -sdk: docker -pinned: false -license: mit -short_description: 'A ' ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/app/api/image-to-text/route.ts b/app/api/image-to-text/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..f960e21abc5798a5099f30927fd692a03022b168 --- /dev/null +++ b/app/api/image-to-text/route.ts @@ -0,0 +1,12 @@ +import { pipeline } from "@huggingface/transformers"; +import { NextRequest, NextResponse } from "next/server"; + +export const POST = async (request: NextRequest) => { + const { url } = await request.json(); + const captioner = await pipeline( + "image-to-text", + "Xenova/vit-gpt2-image-captioning", + ); + const output = await captioner(url); + return NextResponse.json(output); +}; diff --git a/app/api/image/route.ts b/app/api/image/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..d673c93f38f7a947a2e83b8cc3c7cfd17d2960e4 --- /dev/null +++ b/app/api/image/route.ts @@ -0,0 +1,25 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { ListObjectsCommand, PutObjectCommand } from "@aws-sdk/client-s3"; +import { s3 } from "@/lib/s3"; + +const Bucket = "lectus-bucket"; +export async function GET() { + const response = await s3.send(new ListObjectsCommand({ Bucket })); + return NextResponse.json(response?.Contents ?? []); +} + +export async function POST(request: NextRequest) { + try { + const formData = await request.formData(); + const file = formData.get("image") as File; + const Body = (await file.arrayBuffer()) as Buffer; + await s3.send(new PutObjectCommand({ Bucket, Key: file.name, Body })); + + return NextResponse.json({ + key: process.env.AWS_S3_URL + file.name, + }); + } catch (e) { + console.error("error", e); + } +} diff --git a/app/api/route.ts b/app/api/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..dc79a718bac19c734737a29247327927bcbb30c3 --- /dev/null +++ b/app/api/route.ts @@ -0,0 +1,5 @@ +import {NextResponse} from "next/server"; + +export const GET = async () => { + return NextResponse.json({message: "Welcome to visio"}) +}; diff --git a/app/api/speech-to-audio/route.ts b/app/api/speech-to-audio/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..733dab0422c3acbc4fc2fdebbef965a23825f58b --- /dev/null +++ b/app/api/speech-to-audio/route.ts @@ -0,0 +1,16 @@ +import fs from "fs"; +import { NextRequest, NextResponse } from "next/server"; +import { WaveFile } from "wavefile"; +import path from "path"; + +const wav = new WaveFile(); +export const POST = async (request: NextRequest) => { + const { audio, sampling_rate, name } = await request.json(); + console.log(Object.values(audio)); + wav.fromScratch(1, sampling_rate, "32f", Object.values(audio)); + fs.writeFileSync( + path.join(process.cwd() + `/public/${name}.wav`), + wav.toBuffer(), + ); + return NextResponse.json({ audio: `/${name}.wav` }); +}; diff --git a/app/api/text-to-speech-lang/route.ts b/app/api/text-to-speech-lang/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..6b40be37050c5a59cf45988016d16dc571752a46 --- /dev/null +++ b/app/api/text-to-speech-lang/route.ts @@ -0,0 +1,10 @@ +import { pipeline } from "@huggingface/transformers"; +import { NextRequest, NextResponse } from "next/server"; + +export const POST = async (request: NextRequest) => { + const { text, modal } = await request.json(); + const synthesizer = await pipeline("text-to-speech", modal); + const speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/v3.0.0/speaker_embeddings.bin'; + const out = await synthesizer(text, {speaker_embeddings}); + return NextResponse.json(out); +}; diff --git a/app/api/text-to-speech/route.ts b/app/api/text-to-speech/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..1bab1567ba3c876d04a0552b6ed97a696b5faaf3 --- /dev/null +++ b/app/api/text-to-speech/route.ts @@ -0,0 +1,15 @@ +import { pipeline } from "@huggingface/transformers"; + +import { NextRequest, NextResponse } from "next/server"; + +export const POST = async (request: NextRequest) => { + const { text } = await request.json(); + console.log("the text", text); + const synthesizer = await pipeline("text-to-speech", "Xenova/speecht5_tts"); + const speaker_embeddings = + "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin"; + const out = await synthesizer(text, { + speaker_embeddings, + }); + return NextResponse.json(out); +}; diff --git a/app/api/translator/route.ts b/app/api/translator/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..76078be1a80932ae86c399ff63fb23cdb9f3711e --- /dev/null +++ b/app/api/translator/route.ts @@ -0,0 +1,15 @@ +import {pipeline} from "@huggingface/transformers"; +import {NextRequest, NextResponse} from "next/server"; + +export const POST = async (request: NextRequest) => { + const {text, code} = await request.json(); + const translator = await pipeline( + "translation", + "Xenova/m2m100_418M", + ); + const out = await translator(text, { + src_lang: "en", + tgt_lang: code, + } as never); + return NextResponse.json(out); +}; diff --git a/app/constant/index.ts b/app/constant/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..76b2ef9b7263ae9de3bae87ca558bbd423ecc15b --- /dev/null +++ b/app/constant/index.ts @@ -0,0 +1,41 @@ +export const modal = [ + { + lang: "english", + model: "Xenova/mms-tts-eng", + code: "en", + }, + // { + // lang: "tamil", + // model: "Xenova/mms-tts-tam", + // }, + { + lang: "russian", + model: "Xenova/mms-tts-rus", + code: "ru", + }, + { + lang: "hindi", + model: "Xenova/mms-tts-hin", + code: "hi", + }, + { + lang: "spanish", + model: "Xenova/mms-tts-spa", + code: "es", + }, + { + lang: "german", + model: "Xenova/mms-tts-deu", + code: "de", + }, + { + lang: "french", + model: "Xenova/mms-tts-fra", + code: "fr", + }, + { + lang: "portuguese", + model: "Xenova/mms-tts-por", + code: "pt", + }, +]; diff --git a/app/favicon.ico b/app/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..718d6fea4835ec2d246af9800eddb7ffb276240c Binary files /dev/null and b/app/favicon.ico differ diff --git a/app/fonts/GeistMonoVF.woff b/app/fonts/GeistMonoVF.woff new file mode 100644 index 0000000000000000000000000000000000000000..f2ae185cbfd16946a534d819e9eb03924abbcc49 Binary files /dev/null and b/app/fonts/GeistMonoVF.woff differ diff --git a/app/fonts/GeistVF.woff b/app/fonts/GeistVF.woff new file mode 100644 index 0000000000000000000000000000000000000000..1b62daacff96dad6584e71cd962051b82957c313 Binary files /dev/null and b/app/fonts/GeistVF.woff differ diff --git a/app/globals.css b/app/globals.css new file mode 100644 index 0000000000000000000000000000000000000000..aee811bafd07c043edc54528dfb6d97cb5c4ae12 --- /dev/null +++ b/app/globals.css @@ -0,0 +1,75 @@ +@import url('https://api.mapbox.com/mapbox-gl-js/v2.8.1/mapbox-gl.css'); +@tailwind base; +@tailwind components; +@tailwind utilities; + +body { + font-family: Arial, Helvetica, sans-serif; +} + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 240 10% 3.9%; + --card: 0 0% 100%; + --card-foreground: 240 10% 3.9%; + --popover: 0 0% 100%; + --popover-foreground: 240 10% 3.9%; + --primary: 240 5.9% 10%; + --primary-foreground: 0 0% 98%; + --secondary: 240 4.8% 95.9%; + --secondary-foreground: 240 5.9% 10%; + --muted: 240 4.8% 95.9%; + --muted-foreground: 240 3.8% 46.1%; + --accent: 240 4.8% 95.9%; + --accent-foreground: 240 5.9% 10%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 0 0% 98%; + --border: 240 5.9% 90%; + --input: 240 5.9% 90%; + --ring: 240 10% 3.9%; + --chart-1: 12 76% 61%; + --chart-2: 173 58% 39%; + --chart-3: 197 37% 24%; + --chart-4: 43 74% 66%; + --chart-5: 27 87% 67%; + --radius: 0.5rem; + } + + .dark { + --background: 240 10% 3.9%; + --foreground: 0 0% 98%; + --card: 240 10% 3.9%; + --card-foreground: 0 0% 98%; + --popover: 240 10% 3.9%; + --popover-foreground: 0 0% 98%; + --primary: 0 0% 98%; + --primary-foreground: 240 5.9% 10%; + --secondary: 240 3.7% 15.9%; + --secondary-foreground: 0 0% 98%; + --muted: 240 3.7% 15.9%; + --muted-foreground: 240 5% 64.9%; + --accent: 240 3.7% 15.9%; + --accent-foreground: 0 0% 98%; + --destructive: 0 62.8% 30.6%; + --destructive-foreground: 0 0% 98%; + --border: 240 3.7% 15.9%; + --input: 240 3.7% 15.9%; + --ring: 240 4.9% 83.9%; + --chart-1: 220 70% 50%; + --chart-2: 160 60% 45%; + --chart-3: 30 80% 55%; + --chart-4: 280 65% 60%; + --chart-5: 340 75% 55%; + } +} + +@layer base { + * { + @apply border-border; + } + + body { + @apply bg-background text-foreground; + } +} diff --git a/app/layout.tsx b/app/layout.tsx new file mode 100644 index 0000000000000000000000000000000000000000..0521906033f6e6666790f0a0d46170880807686e --- /dev/null +++ b/app/layout.tsx @@ -0,0 +1,39 @@ +import type { Metadata } from "next"; +import localFont from "next/font/local"; +import "./globals.css"; +import { WallProvider } from "./providers/wallprovider"; +import { AsyncProvider } from "./providers/asyncprovider"; + +const geistSans = localFont({ + src: "./fonts/GeistVF.woff", + variable: "--font-geist-sans", + weight: "100 900", +}); +const geistMono = localFont({ + src: "./fonts/GeistMonoVF.woff", + variable: "--font-geist-mono", + weight: "100 900", +}); + +export const metadata: Metadata = { + title: "Create Next App", + description: "Generated by create next app", +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + + + + {children} + + + + ); +} diff --git a/app/page.tsx b/app/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6ffa3a5c8b3604e22ca75ed2dd918f3ea38f0c27 --- /dev/null +++ b/app/page.tsx @@ -0,0 +1,12 @@ +import VisualSearchAssistant from "../components/VisualSearchAssistant"; + +export default function Home() { + return ( +
+

+ Visual Search Assistant +

+ +
+ ); +} diff --git a/app/providers/asyncprovider.tsx b/app/providers/asyncprovider.tsx new file mode 100644 index 0000000000000000000000000000000000000000..42ae6b7f8bba4ac5145a5b224ac1c1a9d65f48ba --- /dev/null +++ b/app/providers/asyncprovider.tsx @@ -0,0 +1,14 @@ +"use client"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; +import React from "react"; + +export const AsyncProvider = ({ children }: { children: React.ReactNode }) => { + const [queryClient] = React.useState(() => new QueryClient()); + return ( + + {children} + + + ); +}; diff --git a/app/providers/wallprovider.tsx b/app/providers/wallprovider.tsx new file mode 100644 index 0000000000000000000000000000000000000000..4c4cca16ded0d1e121872eb812cbc4402aa32170 --- /dev/null +++ b/app/providers/wallprovider.tsx @@ -0,0 +1,8 @@ +"use client"; + +import clsx from "clsx"; +import React from "react"; + +export const WallProvider = ({ children }: { children: React.ReactNode }) => { + return
{children}
; +}; diff --git a/app/services/index.ts b/app/services/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..5b2f7b6d708dcbb806bc117531915b5e051d67df --- /dev/null +++ b/app/services/index.ts @@ -0,0 +1,99 @@ +export const genTextToSpeech = async ({ text }: { text: string }) => { + const response = await fetch("/api/text-to-speech", { + method: "POST", + body: JSON.stringify({ text }), + headers: { + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + throw new Error("Network response was not ok"); + } + const data = await response.json(); + return data as { audio: Float32Array; sampling_rate: number }; +}; + +export const genTextToSpeechLang = async ({ + text, + modal, +}: { + text: string; + modal: string; +}) => { + const response = await fetch("/api/text-to-speech-lang", { + method: "POST", + body: JSON.stringify({ text, modal }), + headers: { + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + throw new Error("Network response was not ok"); + } + const data = await response.json(); + return data as { audio: Float32Array; sampling_rate: number }; +}; + +export const getWavAudio = async ({ + sampling_rate, + audio, + name, +}: { + audio: Float32Array; + sampling_rate: number; + name: string; +}) => { + const response = await fetch("/api/speech-to-audio", { + method: "POST", + body: JSON.stringify({ sampling_rate, audio, name }), + headers: { + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + throw new Error("Network response was not ok"); + } + const data = await response.json(); + return data as { audio: string }; +}; + +export const uploadImage = async ({ form }: { form: FormData }) => { + const response = await fetch("/api/image", { + method: "POST", + body: form, + }); + const json = await response.json(); + return json as { key: string }; +}; + +export const imageToText = async ({ url }: { url: string }) => { + const response = await fetch("/api/image-to-text", { + method: "POST", + body: JSON.stringify({ url }), + headers: { + "Content-Type": "application/json", + }, + }); + return (await response.json()) as Array<{ generated_text: string }>; +}; + +export const getTranslation = async ({ + text, + code, +}: { + text: string; + code: string; +}) => { + const response = await fetch("/api/translator", { + method: "POST", + body: JSON.stringify({ text, code }), + headers: { + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + throw new Error("Network response was not ok"); + } + const data = await response.json(); + return data as Array<{ translation_text: string }>; +}; diff --git a/app/store/index.ts b/app/store/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..f25483e4876361b42e13e956adca55212e841ecb --- /dev/null +++ b/app/store/index.ts @@ -0,0 +1,13 @@ +import { create } from "zustand"; + +interface Steps { + step: number; + increase: (by: number) => void; + decrement: (by: number) => void; +} + +export const useStepsStore = create()((set) => ({ + step: 1, + increase: (by) => set((state) => ({ step: state.step + by })), + decrement: (by) => set((state) => ({ step: state.step - by })), +})); diff --git a/components.json b/components.json new file mode 100644 index 0000000000000000000000000000000000000000..a3128650ce8679561272c5b2dbd0a02fea2ef5b5 --- /dev/null +++ b/components.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": true, + "tsx": true, + "tailwind": { + "config": "tailwind.config.ts", + "css": "app/globals.css", + "baseColor": "zinc", + "cssVariables": true, + "prefix": "" + }, + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + }, + "iconLibrary": "lucide" +} \ No newline at end of file diff --git a/components/AudioPlayback.tsx b/components/AudioPlayback.tsx new file mode 100644 index 0000000000000000000000000000000000000000..4ab51db06fd70c098a9503593ce8f3c9703bc0e6 --- /dev/null +++ b/components/AudioPlayback.tsx @@ -0,0 +1,36 @@ +'use client' + +import { Play, Pause, RotateCcw } from 'lucide-react' +import { Button } from "@/components/ui/button" +import React from "react"; +import {match} from "ts-pattern" + +interface AudioPlaybackProps { + path: string +} + +export default function AudioPlayback({ path }: AudioPlaybackProps) { + const audioRef = React.useRef(null); + + return ( +
+ + + +
+ ) +} + diff --git a/components/DescriptionDisplay.tsx b/components/DescriptionDisplay.tsx new file mode 100644 index 0000000000000000000000000000000000000000..16d62a4996886e3b8f8d10d1f31b43883857617d --- /dev/null +++ b/components/DescriptionDisplay.tsx @@ -0,0 +1,19 @@ +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" + +interface DescriptionDisplayProps { + description: string +} + +export default function DescriptionDisplay({ description }: DescriptionDisplayProps) { + return ( + + + Image Description + + +

{description}

+
+
+ ) +} + diff --git a/components/ImageUpload.tsx b/components/ImageUpload.tsx new file mode 100644 index 0000000000000000000000000000000000000000..15a01b48fddc6da1bce505677caf2b24c491251e --- /dev/null +++ b/components/ImageUpload.tsx @@ -0,0 +1,130 @@ +import React from "react"; +import {Camera, Upload} from "lucide-react"; +import {Button} from "@/components/ui/button"; + +interface ImageUploadProps { + onImageUpload: (file: File) => void; + sideEffect: (text: string | null) => void; +} + +export default function ImageUpload({onImageUpload, sideEffect}: ImageUploadProps) { + const [isDragging, setIsDragging] = React.useState(false); + const fileInputRef = React.useRef(null); + const videoRef = React.useRef(null); + const [isCameraActive, setIsCameraActive] = React.useState(false); + + const handleDragOver = (e: React.DragEvent) => { + e.preventDefault(); + setIsDragging(true); + }; + + const handleDragLeave = () => { + setIsDragging(false); + }; + + const handleDrop = (e: React.DragEvent) => { + e.preventDefault(); + setIsDragging(false); + const file = e.dataTransfer.files[0]; + if (file) { + onImageUpload(file); + } + }; + + const handleFileInput = (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) { + onImageUpload(file); + } + }; + + const activateCamera = async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({video: true}); + if (videoRef.current) { + videoRef.current.srcObject = stream; + setIsCameraActive(true); + } + } catch (error) { + console.error("Error accessing camera:", error); + } + }; + + const capturePhoto = () => { + if (videoRef.current) { + const canvas = document.createElement("canvas"); + canvas.width = videoRef.current.videoWidth; + canvas.height = videoRef.current.videoHeight; + canvas.getContext("2d")?.drawImage(videoRef.current, 0, 0); + canvas.toBlob((blob) => { + if (blob) { + const file = new File([blob], "camera-photo.jpg", { + type: "image/jpeg", + }); + onImageUpload(file); + } + }, "image/jpeg"); + setIsCameraActive(false); + } + }; + + return ( +
+ {isCameraActive ? ( +
+
+ ) : ( + <> +
event.preventDefault()} + encType="multipart/form-data" + > +

+ Drag and drop an image here, or click to select a file +

+ { + handleFileInput(event); + sideEffect(null) + }} + className="hidden" + ref={fileInputRef} + /> +
+ + +
+
+ + )} +
+ ); +} diff --git a/components/LanguageSelector.tsx b/components/LanguageSelector.tsx new file mode 100644 index 0000000000000000000000000000000000000000..39aece030c160cd01fae6198a59c7f7afade10c3 --- /dev/null +++ b/components/LanguageSelector.tsx @@ -0,0 +1,41 @@ +import {modal} from "@/app/constant"; +import {Select, SelectContent, SelectItem, SelectTrigger, SelectValue,} from "@/components/ui/select"; + +interface LanguageSelectorProps { + onLanguageChange: (language: string) => void; + sideEffect: (text: string | null) => void +} + +export default function LanguageSelector({ + onLanguageChange, + sideEffect, + }: LanguageSelectorProps) { + return ( +
+ + +
+ ); +} diff --git a/components/VisualSearchAssistant.tsx b/components/VisualSearchAssistant.tsx new file mode 100644 index 0000000000000000000000000000000000000000..eb2dce7e633159174495f6246466a247dc02fd8d --- /dev/null +++ b/components/VisualSearchAssistant.tsx @@ -0,0 +1,126 @@ +"use client"; + +import React from "react"; +import ImageUpload from "./ImageUpload"; +import DescriptionDisplay from "./DescriptionDisplay"; +import AudioPlayback from "./AudioPlayback"; +import {Card, CardContent} from "@/components/ui/card"; +import {useGenWav, useImageToText, useTextToSpeachLang, useTranslator, useUploadImage} from "@/hooks/api"; +import LanguageSelector from "./LanguageSelector"; +import {modal} from "@/app/constant"; +import Image from "next/image"; + +export default function VisualSearchAssistant() { + const [imageUrl, setImageUrl] = React.useState(null); + const [description, setDescription] = React.useState(""); + const [language, setLanguage] = React.useState("Xenova/mms-tts-eng"); + const [path, setPath] = React.useState(null); + const code = modal.find((lang) => lang.model === language)?.code || "en"; + const upload = useUploadImage(); + const iToText = useImageToText(); + const translator = useTranslator(); + const tts = useTextToSpeachLang(); + const wav = useGenWav(); + + const handleImageUpload = async (file: File) => { + const url = URL.createObjectURL(file); + setImageUrl(url); + + const formData = new FormData(); + formData.append("image", file); + + try { + upload.mutate( + {form: formData}, + { + onSuccess: (data) => { + iToText.mutate( + {url: data?.key}, + { + onSuccess: (data) => { + translator.mutate( + {text: data[0].generated_text, code}, + { + onSuccess: (data) => { + console.log("data-desc", data); + setDescription(data[0]?.translation_text); + console.log("lang", language, data); + tts.mutate({text: data[0]?.translation_text, modal: language}, { + onSuccess: (data) => { + wav.mutate({ + sampling_rate: data.sampling_rate, + audio: data.audio, + name: crypto.randomUUID() + }, { + onSuccess: (data) => { + setPath(data.audio); + } + }) + } + }) + }, + }, + ); + }, + }, + ); + }, + }, + ); + } catch (error) { + console.error("Error processing image:", error); + setDescription("Failed to process image. Please try again."); + } + }; + console.log("upload data", upload?.data); + + const renderStatusMessage = () => { + if (upload.isPending) { + return

Please wait while we process your image...

; + } + if (iToText.isPending) { + return

Processing image and extracting information, please wait...

; + } + if (translator.isPending) { + return

Wait while we translate your text...

; + } + if (tts.isPending) { + return

Wait while we process the audio...

; + } + if (wav.isPending) { + return

Generating audio file, please wait...

; + } + return null; + }; + + + return ( +
+ + + + + {imageUrl && ( +
+ Uploaded image +
+ )} + {renderStatusMessage()} + {description && ( +
+ + +
+ )} +
+
+
+ ); +} diff --git a/components/footer.tsx b/components/footer.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3afdf975b2679b89e3bfe123e5204921fad935dd --- /dev/null +++ b/components/footer.tsx @@ -0,0 +1,26 @@ +"use client"; +import {Button} from "./ui/button"; +import React from "react"; + +export const Footer = () => { + return ( +
+
+
+ + +
+
+
+ ) +} diff --git a/components/header.tsx b/components/header.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e97eb4130166dc5fa082b9df701916f72e87e5b7 --- /dev/null +++ b/components/header.tsx @@ -0,0 +1,34 @@ +"use client"; +import { Volume2, VolumeX } from "lucide-react"; +import { Button } from "./ui/button"; +import React from "react"; + +export const Header = () => { + const [isMuted, setIsMuted] = React.useState(false); + const toggleMute = () => { + setIsMuted(!isMuted); + }; + return ( +
+
+
+ +
+ +
+
+
+ ); +}; diff --git a/components/ui/button.tsx b/components/ui/button.tsx new file mode 100644 index 0000000000000000000000000000000000000000..65d4fcd9ca74240125c5f72cf84c873781141fea --- /dev/null +++ b/components/ui/button.tsx @@ -0,0 +1,57 @@ +import * as React from "react" +import { Slot } from "@radix-ui/react-slot" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const buttonVariants = cva( + "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0", + { + variants: { + variant: { + default: + "bg-primary text-primary-foreground shadow hover:bg-primary/90", + destructive: + "bg-destructive text-destructive-foreground shadow-sm hover:bg-destructive/90", + outline: + "border border-input bg-background shadow-sm hover:bg-accent hover:text-accent-foreground", + secondary: + "bg-secondary text-secondary-foreground shadow-sm hover:bg-secondary/80", + ghost: "hover:bg-accent hover:text-accent-foreground", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: "h-9 px-4 py-2", + sm: "h-8 rounded-md px-3 text-xs", + lg: "h-10 rounded-md px-8", + icon: "h-9 w-9", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + +export interface ButtonProps + extends React.ButtonHTMLAttributes, + VariantProps { + asChild?: boolean +} + +const Button = React.forwardRef( + ({ className, variant, size, asChild = false, ...props }, ref) => { + const Comp = asChild ? Slot : "button" + return ( + + ) + } +) +Button.displayName = "Button" + +export { Button, buttonVariants } diff --git a/components/ui/card.tsx b/components/ui/card.tsx new file mode 100644 index 0000000000000000000000000000000000000000..cabfbfc59d955db9efc2049783cad4a71db55442 --- /dev/null +++ b/components/ui/card.tsx @@ -0,0 +1,76 @@ +import * as React from "react" + +import { cn } from "@/lib/utils" + +const Card = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +Card.displayName = "Card" + +const CardHeader = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardHeader.displayName = "CardHeader" + +const CardTitle = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardTitle.displayName = "CardTitle" + +const CardDescription = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardDescription.displayName = "CardDescription" + +const CardContent = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardContent.displayName = "CardContent" + +const CardFooter = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +CardFooter.displayName = "CardFooter" + +export { Card, CardHeader, CardFooter, CardTitle, CardDescription, CardContent } diff --git a/components/ui/dialog.tsx b/components/ui/dialog.tsx new file mode 100644 index 0000000000000000000000000000000000000000..1647513eced8f9e788c0e8b63c99654d3a542a9c --- /dev/null +++ b/components/ui/dialog.tsx @@ -0,0 +1,122 @@ +"use client" + +import * as React from "react" +import * as DialogPrimitive from "@radix-ui/react-dialog" +import { X } from "lucide-react" + +import { cn } from "@/lib/utils" + +const Dialog = DialogPrimitive.Root + +const DialogTrigger = DialogPrimitive.Trigger + +const DialogPortal = DialogPrimitive.Portal + +const DialogClose = DialogPrimitive.Close + +const DialogOverlay = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogOverlay.displayName = DialogPrimitive.Overlay.displayName + +const DialogContent = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, children, ...props }, ref) => ( + + + + {children} + + + Close + + + +)) +DialogContent.displayName = DialogPrimitive.Content.displayName + +const DialogHeader = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +DialogHeader.displayName = "DialogHeader" + +const DialogFooter = ({ + className, + ...props +}: React.HTMLAttributes) => ( +
+) +DialogFooter.displayName = "DialogFooter" + +const DialogTitle = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogTitle.displayName = DialogPrimitive.Title.displayName + +const DialogDescription = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => ( + +)) +DialogDescription.displayName = DialogPrimitive.Description.displayName + +export { + Dialog, + DialogPortal, + DialogOverlay, + DialogTrigger, + DialogClose, + DialogContent, + DialogHeader, + DialogFooter, + DialogTitle, + DialogDescription, +} diff --git a/components/ui/form.tsx b/components/ui/form.tsx new file mode 100644 index 0000000000000000000000000000000000000000..b6daa654a177fa31b3d84e1af645ba874fc43762 --- /dev/null +++ b/components/ui/form.tsx @@ -0,0 +1,178 @@ +"use client" + +import * as React from "react" +import * as LabelPrimitive from "@radix-ui/react-label" +import { Slot } from "@radix-ui/react-slot" +import { + Controller, + ControllerProps, + FieldPath, + FieldValues, + FormProvider, + useFormContext, +} from "react-hook-form" + +import { cn } from "@/lib/utils" +import { Label } from "@/components/ui/label" + +const Form = FormProvider + +type FormFieldContextValue< + TFieldValues extends FieldValues = FieldValues, + TName extends FieldPath = FieldPath +> = { + name: TName +} + +const FormFieldContext = React.createContext( + {} as FormFieldContextValue +) + +const FormField = < + TFieldValues extends FieldValues = FieldValues, + TName extends FieldPath = FieldPath +>({ + ...props +}: ControllerProps) => { + return ( + + + + ) +} + +const useFormField = () => { + const fieldContext = React.useContext(FormFieldContext) + const itemContext = React.useContext(FormItemContext) + const { getFieldState, formState } = useFormContext() + + const fieldState = getFieldState(fieldContext.name, formState) + + if (!fieldContext) { + throw new Error("useFormField should be used within ") + } + + const { id } = itemContext + + return { + id, + name: fieldContext.name, + formItemId: `${id}-form-item`, + formDescriptionId: `${id}-form-item-description`, + formMessageId: `${id}-form-item-message`, + ...fieldState, + } +} + +type FormItemContextValue = { + id: string +} + +const FormItemContext = React.createContext( + {} as FormItemContextValue +) + +const FormItem = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => { + const id = React.useId() + + return ( + +
+ + ) +}) +FormItem.displayName = "FormItem" + +const FormLabel = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef +>(({ className, ...props }, ref) => { + const { error, formItemId } = useFormField() + + return ( +