feat(new tool): Math OCR to Latex

FIx #1009

src/tools/index.ts

@@ -2,6 +2,7 @@ import { tool as base64FileConverter } from './base64-file-converter';
 import { tool as base64StringConverter } from './base64-string-converter';
 import { tool as basicAuthGenerator } from './basic-auth-generator';
 import { tool as emailNormalizer } from './email-normalizer';
+import { tool as mathOcr } from './math-ocr';
 
 import { tool as asciiTextDrawer } from './ascii-text-drawer';
 
@@ -162,7 +163,12 @@ export const toolsByCategory: ToolCategory[] = [
   },
   {
     name: 'Math',
-    components: [mathEvaluator, etaCalculator, percentageCalculator],
+    components: [
+      mathEvaluator,
+      etaCalculator,
+      percentageCalculator,
+      mathOcr,
+    ],
   },
   {
     name: 'Measurement',

src/tools/math-ocr/index.ts

@@ -0,0 +1,12 @@
+import { MathSymbols } from '@vicons/tabler';
+import { defineTool } from '../tool';
+
+export const tool = defineTool({
+  name: 'Math OCR',
+  path: '/math-ocr',
+  description: 'Convert Math Formula images to Latex',
+  keywords: ['math', 'ocr', 'latex', 'formula', 'image'],
+  component: () => import('./math-ocr.vue'),
+  icon: MathSymbols,
+  createdAt: new Date('2024-08-15'),
+});

src/tools/math-ocr/math-ocr.vue

@@ -0,0 +1,97 @@
+<script setup lang="ts">
+import type { Ref } from 'vue';
+import { pipeline } from '@huggingface/transformers';
+import VueMathjax from 'vue-mathjax-next';
+import { useScriptTag } from '@vueuse/core';
+import TextareaCopyable from '@/components/TextareaCopyable.vue';
+
+useScriptTag('https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/MathJax.js?config=TeX-AMS_HTML');
+
+const ocrInProgress = ref(false);
+const fileInput = ref() as Ref<File>;
+const latexResult = computedAsync(async () => {
+  try {
+    return (await ocr(fileInput.value));
+  }
+  catch (e: any) {
+    return e.toString();
+  }
+});
+
+async function onUpload(file: File) {
+  if (file) {
+    fileInput.value = file;
+  }
+}
+
+function toBase64(file: File) {
+  return new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = () => resolve(reader.result?.toString() ?? '');
+    reader.onerror = error => reject(error);
+  });
+}
+
+type TexifyResult = Array<{ generated_text: string }>;
+
+async function ocr(file: File) {
+  if (!file) {
+    return '';
+  }
+  ocrInProgress.value = true;
+
+  const imgBase64 = await toBase64(file);
+
+  const pipe = await pipeline('image-to-text', 'Xenova/texify');
+  const out: TexifyResult = (await pipe(imgBase64, { max_new_tokens: 384 })) as TexifyResult;
+
+  ocrInProgress.value = false;
+  return out.map(t => t.generated_text).join('\n');
+};
+</script>
+
+<template>
+  <div style="max-width: 600px;">
+    <c-alert type="warning" mb-2>
+      NB: processing is done in your browser, so be patient, processing can take a while
+      <br>
+      This tool required internet connection (to access models)
+    </c-alert>
+
+    <c-file-upload
+      title="Drag and drop a Image here, or click to select a file"
+      paste-image
+      @file-upload="onUpload"
+    />
+
+    <n-divider />
+
+    <div>
+      <h3>Latex Result</h3>
+      <TextareaCopyable
+        v-if="!ocrInProgress"
+        v-model:value="latexResult"
+        :word-wrap="true" mb-2
+      />
+
+      <div style="text-align: center">
+        <VueMathjax
+          v-if="!ocrInProgress"
+          :formula="latexResult"
+        />
+      </div>
+
+      <n-spin
+        v-if="ocrInProgress"
+        size="small"
+      />
+    </div>
+  </div>
+</template>
+
+<style lang="less" scoped>
+::v-deep(.n-upload-trigger) {
+  width: 100%;
+}
+</style>

src/tools/math-ocr/vue-mathjax-next.d.ts

@@ -0,0 +1,3 @@
+declare module "vue-mathjax-next" {
+    
+}

src/ui/c-file-upload/c-file-upload.vue

@@ -5,10 +5,12 @@ const props = withDefaults(defineProps<{
   multiple?: boolean
   accept?: string
   title?: string
+  pasteImage?: boolean
 }>(), {
   multiple: false,
   accept: undefined,
   title: 'Drag and drop files here, or click to select files',
+  pasteImage: false,
 });
 
 const emit = defineEmits<{
@@ -16,11 +18,31 @@ const emit = defineEmits<{
   (event: 'fileUpload', file: File): void
 }>();
 
-const { multiple } = toRefs(props);
+const { multiple, pasteImage } = toRefs(props);
 
 const isOverDropZone = ref(false);
 
+function toBase64(file: File) {
+  return new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = () => resolve(reader.result?.toString() ?? '');
+    reader.onerror = error => reject(error);
+  });
+}
+
 const fileInput = ref<HTMLInputElement | null>(null);
+const imgPreview = ref<HTMLImageElement | null>(null);
+async function handlePreview(image: File) {
+  if (imgPreview.value) {
+    imgPreview.value.src = await toBase64(image);
+  }
+}
+function clearPreview() {
+  if (imgPreview.value) {
+    imgPreview.value.src = '';
+  }
+}
 
 function triggerFileInput() {
   fileInput.value?.click();
@@ -39,7 +61,30 @@ function handleDrop(event: DragEvent) {
   handleUpload(files);
 }
 
-function handleUpload(files: FileList | null | undefined) {
+async function onPasteImage(evt: ClipboardEvent) {
+  if (!pasteImage.value) {
+    return false;
+  }
+
+  const items = evt.clipboardData?.items;
+  if (!items) {
+    return false;
+  }
+  for (let i = 0; i < items.length; i++) {
+    if (items[i].type.includes('image')) {
+      const imageFile = items[i].getAsFile();
+      if (imageFile) {
+        await handlePreview(imageFile);
+        emit('fileUpload', imageFile);
+      }
+    }
+  }
+  return true;
+}
+
+async function handleUpload(files: FileList | null | undefined) {
+  clearPreview();
+
   if (_.isNil(files) || _.isEmpty(files)) {
     return;
   }
@@ -49,6 +94,7 @@ function handleUpload(files: FileList | null | undefined) {
     return;
   }
 
+  await handlePreview(files[0]);
   emit('fileUpload', files[0]);
 }
 </script>
@@ -60,6 +106,7 @@ function handleUpload(files: FileList | null | undefined) {
       'border-primary border-opacity-100': isOverDropZone,
     }"
     @click="triggerFileInput"
+    @paste.prevent="onPasteImage"
     @drop.prevent="handleDrop"
     @dragover.prevent
     @dragenter="isOverDropZone = true"
@@ -73,6 +120,7 @@ function handleUpload(files: FileList | null | undefined) {
       :accept="accept"
       @change="handleFileInput"
     >
+
     <slot>
       <span op-70>
         {{ title }}
@@ -90,6 +138,22 @@ function handleUpload(files: FileList | null | undefined) {
       <c-button>
         Browse files
       </c-button>
+
+      <div v-if="pasteImage">
+        <!-- separator -->
+        <div my-4 w-full flex items-center justify-center op-70>
+          <div class="h-1px max-w-100px flex-1 bg-gray-300 op-50" />
+          <div class="mx-2 text-gray-400">
+            or
+          </div>
+          <div class="h-1px max-w-100px flex-1 bg-gray-300 op-50" />
+        </div>
+
+        <p>Paste an image from clipboard</p>
+      </div>
     </slot>
+    <div mt-2>
+      <img ref="imgPreview" width="150">
+    </div>
   </div>
 </template>