Skip to content

Commit 50c0ddc

Browse files
mishig25mishig25
committed
Implemend PDF-chat feature
1 parent c01d7bc commit 50c0ddc

30 files changed

+864
-68
lines changed

‎package-lock.json

Lines changed: 366 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
"nanoid": "^4.0.2",
6262
"openid-client": "^5.4.2",
6363
"parquetjs": "^0.11.2",
64+
"pdfjs-dist": "^4.0.269",
6465
"postcss": "^8.4.31",
6566
"serpapi": "^1.1.1",
6667
"tailwind-scrollbar": "^3.0.0",

‎src/lib/buildPrompt.ts

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ import type { BackendModel } from "./server/models";
22
import type { Message } from "./types/Message";
33
import { format } from "date-fns";
44
import type { WebSearch } from "./types/WebSearch";
5-
import { downloadFile } from "./server/files/downloadFile";
5+
import type { PdfSearch } from "./types/PdfSearch";
6+
import { downloadImgFile } from "./server/files/downloadFile";
67
import type { Conversation } from "./types/Conversation";
78

89
interface buildPromptOptions {
@@ -11,6 +12,7 @@ interface buildPromptOptions {
1112
model: BackendModel;
1213
locals?: App.Locals;
1314
webSearch?: WebSearch;
15+
pdfSearch?: PdfSearch;
1416
preprompt?: string;
1517
files?: File[];
1618
}
@@ -19,6 +21,7 @@ export async function buildPrompt({
1921
messages,
2022
model,
2123
webSearch,
24+
pdfSearch,
2225
preprompt,
2326
id,
2427
}: buildPromptOptions): Promise<string> {
@@ -47,6 +50,31 @@ export async function buildPrompt({
4750
`,
4851
},
4952
];
53+
}else if (pdfSearch && pdfSearch.context) {
54+
const lastMsg = messages.slice(-1)[0];
55+
const messagesWithoutLastUsrMsg = messages.slice(0, -1);
56+
const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
57+
58+
const previousQuestions =
59+
previousUserMessages.length > 0
60+
? `Previous questions: \n${previousUserMessages
61+
.map(({ content }) => `- ${content}`)
62+
.join("\n")}`
63+
: "";
64+
65+
messages = [
66+
...messagesWithoutLastUsrMsg,
67+
{
68+
from: "user",
69+
content: `Below are the information I extracted from a PDF file that might be useful:
70+
=====================
71+
${pdfSearch.context}
72+
=====================
73+
${previousQuestions}
74+
Answer the question: ${lastMsg.content}
75+
`,
76+
},
77+
];
5078
}
5179

5280
// section to handle potential files input
@@ -60,7 +88,7 @@ export async function buildPrompt({
6088
const markdowns = await Promise.all(
6189
el.files.map(async (hash) => {
6290
try {
63-
const { content: image, mime } = await downloadFile(hash, id);
91+
const { content: image, mime } = await downloadImgFile(hash, id);
6492
const b64 = image.toString("base64");
6593
return `![](data:${mime};base64,${b64})})`;
6694
} catch (e) {

‎src/lib/components/OpenPdfSearchResults.svelte

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
<script lang="ts">
2+
import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
3+
import CarbonCaretRight from "~icons/carbon/caret-right";
4+
5+
import CarbonCheckmark from "~icons/carbon/checkmark-filled";
6+
import CarbonError from "~icons/carbon/error-filled";
7+
8+
import EosIconsLoading from "~icons/eos-icons/loading";
9+
10+
export let loading = false;
11+
export let classNames = "";
12+
export let pdfSearchMessages: PdfSearchUpdate[] = [];
13+
14+
let detailsOpen: boolean;
15+
let error: boolean;
16+
$: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
17+
</script>
18+
19+
<details
20+
class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
21+
bind:open={detailsOpen}
22+
>
23+
<summary
24+
class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
25+
>
26+
{#if error}
27+
<CarbonError class="my-auto text-red-700 dark:text-red-500" />
28+
{:else if loading}
29+
<EosIconsLoading class="my-auto text-gray-500" />
30+
{:else}
31+
<CarbonCheckmark class="my-auto text-gray-500" />
32+
{/if}
33+
<span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
34+
>PDF search
35+
</span>
36+
<div class="my-auto transition-all" class:rotate-90={detailsOpen}>
37+
<CarbonCaretRight />
38+
</div>
39+
</summary>
40+
41+
<div class="content px-5 pb-5 pt-4">
42+
{#if pdfSearchMessages.length === 0}
43+
<div class="mx-auto w-fit">
44+
<EosIconsLoading class="mb-3 h-4 w-4" />
45+
</div>
46+
{:else}
47+
<ol>
48+
{#each pdfSearchMessages as message}
49+
{#if message.messageType === "update"}
50+
<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
51+
<div class="flex items-start">
52+
<div
53+
class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
54+
? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
55+
: ''}"
56+
/>
57+
<h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
58+
{message.message}
59+
</h3>
60+
</div>
61+
{#if message.args}
62+
<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
63+
{message.args}
64+
</p>
65+
{/if}
66+
</li>
67+
{:else if message.messageType === "error"}
68+
<li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
69+
<div class="flex items-start">
70+
<CarbonError
71+
class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
72+
/>
73+
<h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
74+
{message.message}
75+
</h3>
76+
</div>
77+
{#if message.args}
78+
<p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
79+
{message.args}
80+
</p>
81+
{/if}
82+
</li>
83+
{/if}
84+
{/each}
85+
</ol>
86+
{/if}
87+
</div>
88+
</details>
89+
90+
<style>
91+
@keyframes grow {
92+
0% {
93+
font-size: 0;
94+
opacity: 0;
95+
}
96+
30% {
97+
font-size: 1em;
98+
opacity: 0;
99+
}
100+
100% {
101+
opacity: 1;
102+
}
103+
}
104+
105+
details[open] .content {
106+
animation-name: grow;
107+
animation-duration: 300ms;
108+
animation-delay: 0ms;
109+
}
110+
111+
details summary::-webkit-details-marker {
112+
display: none;
113+
}
114+
</style>

‎src/lib/components/UploadBtn.svelte

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,52 @@
11
<script lang="ts">
2+
import {createEventDispatcher} from "svelte";
23
import CarbonUpload from "~icons/carbon/upload";
34
45
export let classNames = "";
6+
export let multimodal = false;
57
export let files: File[];
6-
let filelist: FileList;
8+
export let uploadingPdf = false;
9+
10+
const accept = multimodal ? "image/*,.pdf" : ".pdf";
11+
const label = multimodal ? "Upload image or PDF" : "Upload PDF";
12+
let fileInput: HTMLInputElement;
713
8-
$: if (filelist) {
9-
files = Array.from(filelist);
14+
const dispatch = createEventDispatcher<{
15+
uploadpdf: File;
16+
}>();
17+
18+
function onChange() {
19+
if(!fileInput.files){
20+
return;
21+
}
22+
23+
const file = fileInput.files?.[0];
24+
if (file?.type === "application/pdf") {
25+
// pdf upload
26+
dispatch("uploadpdf", file);
27+
}else{
28+
// image files for multimodal models
29+
files = Array.from(fileInput.files);
30+
}
1031
}
1132
</script>
1233

1334
<button
1435
class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
36+
class:animate-pulse={uploadingPdf}
37+
class:pointer-events-none={uploadingPdf}
1538
>
1639
<input
17-
bind:files={filelist}
40+
bind:this={fileInput}
41+
on:change={onChange}
1842
class="absolute w-full cursor-pointer opacity-0"
1943
type="file"
20-
accept="image/*"
44+
{accept}
2145
/>
22-
<CarbonUpload class="mr-2 text-xs " /> Upload image
46+
<CarbonUpload class="mr-2 text-xs " />
47+
{#if uploadingPdf}
48+
Processing PDF file
49+
{:else}
50+
{label}
51+
{/if}
2352
</button>

‎src/lib/components/chat/ChatMessage.svelte

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717
import type { Model } from "$lib/types/Model";
1818
1919
import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
20-
import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
20+
import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
21+
import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
2122
2223
function sanitizeMd(md: string) {
2324
let ret = md
@@ -49,7 +50,7 @@
4950
export let readOnly = false;
5051
export let isTapped = false;
5152
52-
export let webSearchMessages: WebSearchUpdate[];
53+
export let RAGMessages: RAGUpdate[];
5354
5455
const dispatch = createEventDispatcher<{
5556
retry: { content: string; id: Message["id"] };
@@ -108,9 +109,15 @@
108109
109110
let searchUpdates: WebSearchUpdate[] = [];
110111
111-
$: searchUpdates = ((webSearchMessages.length > 0
112-
? webSearchMessages
112+
$: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
113+
? RAGMessages.filter(({type}) => type === "webSearch")
113114
: message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
115+
116+
let pdfUpdates: PdfSearchUpdate[] = [];
117+
118+
$: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
119+
? RAGMessages.filter(({type}) => type === "pdfSearch")
120+
: message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
114121
115122
$: downloadLink =
116123
message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
@@ -153,7 +160,14 @@
153160
loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
154161
/>
155162
{/if}
156-
{#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
163+
{#if pdfUpdates && pdfUpdates.length > 0}
164+
<OpenPdfSearchResults
165+
classNames={tokens.length ? "mb-3.5" : ""}
166+
pdfSearchMessages={pdfUpdates}
167+
loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
168+
/>
169+
{/if}
170+
{#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
157171
<IconLoading />
158172
{/if}
159173

‎src/lib/components/chat/ChatMessages.svelte

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import type { LayoutData } from "../../../routes/$types";
99
import ChatIntroduction from "./ChatIntroduction.svelte";
1010
import ChatMessage from "./ChatMessage.svelte";
11-
import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
11+
import type { RAGUpdate } from "$lib/types/MessageUpdate";
1212
import { browser } from "$app/environment";
1313
import SystemPromptModal from "../SystemPromptModal.svelte";
1414
import { page } from "$app/stores";
@@ -25,7 +25,7 @@
2525
2626
let chatContainer: HTMLElement;
2727
28-
export let webSearchMessages: WebSearchUpdate[] = [];
28+
export let RAGMessages: RAGUpdate[] = [];
2929
3030
async function scrollToBottom() {
3131
await tick();
@@ -40,7 +40,7 @@
4040

4141
<div
4242
class="scrollbar-custom mr-1 h-full overflow-y-auto"
43-
use:snapScrollToBottom={messages.length ? [...messages, ...webSearchMessages] : false}
43+
use:snapScrollToBottom={messages.length ? [...messages, ...RAGMessages] : false}
4444
bind:this={chatContainer}
4545
>
4646
<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
@@ -54,7 +54,7 @@
5454
{isAuthor}
5555
{readOnly}
5656
model={currentModel}
57-
webSearchMessages={i === messages.length - 1 ? webSearchMessages : []}
57+
RAGMessages={i === messages.length - 1 ? RAGMessages : []}
5858
on:retry
5959
on:vote
6060
/>
@@ -65,7 +65,7 @@
6565
<ChatMessage
6666
message={{ from: "assistant", content: "", id: randomUUID() }}
6767
model={currentModel}
68-
{webSearchMessages}
68+
{RAGMessages}
6969
/>
7070
{/if}
7171
<div class="h-44 flex-none" />

‎src/lib/components/chat/ChatWindow.svelte

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import type { Model } from "$lib/types/Model";
1616
import WebSearchToggle from "../WebSearchToggle.svelte";
1717
import LoginModal from "../LoginModal.svelte";
18-
import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
18+
import type { RAGUpdate } from "$lib/types/MessageUpdate";
1919
import { page } from "$app/stores";
2020
import DisclaimerModal from "../DisclaimerModal.svelte";
2121
import FileDropzone from "./FileDropzone.svelte";
@@ -30,9 +30,10 @@
3030
export let shared = false;
3131
export let currentModel: Model;
3232
export let models: Model[];
33-
export let webSearchMessages: WebSearchUpdate[] = [];
33+
export let RAGMessages: RAGUpdate[] = [];
3434
export let preprompt: string | undefined = undefined;
3535
export let files: File[] = [];
36+
export let uploadingPdf: boolean = false;
3637
3738
$: isReadOnly = !models.some((model) => model.id === currentModel.id);
3839
@@ -94,7 +95,7 @@
9495
{messages}
9596
readOnly={isReadOnly}
9697
isAuthor={!shared}
97-
{webSearchMessages}
98+
{RAGMessages}
9899
{preprompt}
99100
on:message={(ev) => {
100101
if ($page.data.loginRequired) {
@@ -153,9 +154,8 @@
153154
content: messages[messages.length - 1].content,
154155
})}
155156
/>
156-
{:else if currentModel.multimodal}
157-
<UploadBtn bind:files classNames="ml-auto" />
158157
{/if}
158+
<UploadBtn bind:files on:uploadpdf classNames="ml-auto" multimodal={currentModel.multimodal} {uploadingPdf} />
159159
</div>
160160
<form
161161
on:dragover={onDragOver}

0 commit comments

Comments
 (0)