Skip to content

Commit b8e6e9c

Browse files
andrew-polkandrew-polk
authored
Merge pull request #116 from sillsdev/DownloadVideo
fix: Download Notion-hosted videos so the links don't expire (#110)
2 parents f1176c5 + ea4e865 commit b8e6e9c

File tree

5 files changed

+174
-58
lines changed

5 files changed

+174
-58
lines changed

‎src/assets.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import * as fs from "fs-extra";
2+
import * as Path from "path";
3+
import { verbose } from "./log";
4+
5+
export enum AssetType {
6+
Image = "image",
7+
Video = "video",
8+
}
9+
10+
export function writeAsset(path: string, buffer: Buffer): void {
11+
// Note: it's tempting to not spend time writing this out if we already have
12+
// it from a previous run. But we don't really know it's the same. A) it
13+
// could just have the same name, B) it could have been previously
14+
// unlocalized and thus filled with a copy of the primary language image
15+
// while and now is localized.
16+
if (fs.pathExistsSync(path)) {
17+
verbose("Replacing asset " + path);
18+
} else {
19+
verbose("Adding asset " + path);
20+
fs.mkdirsSync(Path.dirname(path));
21+
}
22+
fs.createWriteStream(path).write(buffer); // async but we're not waiting
23+
}

‎src/images.ts‎

Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import * as fs from "fs-extra";
22
import FileType, { FileTypeResult } from "file-type";
33
import axios from "axios";
4-
import * as Path from "path";
54
import { makeImagePersistencePlan } from "./MakeImagePersistencePlan";
65
import { warning, logDebug, verbose, info } from "./log";
76
import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
@@ -10,6 +9,7 @@ import {
109
IDocuNotionContextPageInfo,
1110
IPlugin,
1211
} from "./plugins/pluginTypes";
12+
import { writeAsset } from "./assets";
1313

1414
// We handle several things here:
1515
// 1) copy images locally instead of leaving them in Notion
@@ -158,11 +158,13 @@ async function readPrimaryImage(imageSet: ImageSet) {
158158
}
159159

160160
async function saveImage(imageSet: ImageSet): Promise<void> {
161-
writeImageIfNew(imageSet.primaryFileOutputPath!, imageSet.primaryBuffer!);
161+
const path = imageSet.primaryFileOutputPath!;
162+
imageWasSeen(path);
163+
writeAsset(path, imageSet.primaryBuffer!);
162164

163165
for (const localizedImage of imageSet.localizedUrls) {
164166
let buffer = imageSet.primaryBuffer!;
165-
// if we have a urls for the localized screenshot, download it
167+
// if we have a url for the localized screenshot, download it
166168
if (localizedImage?.url.length > 0) {
167169
verbose(`Retrieving ${localizedImage.iso632Code} version...`);
168170
const response = await fetch(localizedImage.url);
@@ -180,30 +182,15 @@ async function saveImage(imageSet: ImageSet): Promise<void> {
180182
imageSet.pageInfo!.relativeFilePathToFolderContainingPage
181183
}`;
182184

183-
writeImageIfNew(
184-
(directory + "/" + imageSet.outputFileName!).replaceAll("//", "/"),
185-
buffer
185+
const newPath = (directory + "/" + imageSet.outputFileName!).replaceAll(
186+
"//",
187+
"/"
186188
);
189+
imageWasSeen(newPath);
190+
writeAsset(newPath, buffer);
187191
}
188192
}
189193

190-
function writeImageIfNew(path: string, buffer: Buffer) {
191-
imageWasSeen(path);
192-
193-
// Note: it's tempting to not spend time writing this out if we already have
194-
// it from a previous run. But we don't really know it's the same. A) it
195-
// could just have the same name, B) it could have been previously
196-
// unlocalized and thus filled with a copy of the primary language image
197-
// while and now is localized.
198-
if (fs.pathExistsSync(path)) {
199-
verbose("Replacing image " + path);
200-
} else {
201-
verbose("Adding image " + path);
202-
fs.mkdirsSync(Path.dirname(path));
203-
}
204-
fs.createWriteStream(path).write(buffer); // async but we're not waiting
205-
}
206-
207194
export function parseImageBlock(image: any): ImageSet {
208195
if (!locales) throw Error("Did you call initImageHandling()?");
209196
const imageSet: ImageSet = {

‎src/plugins/VideoTransformer.spec.ts‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,28 @@
1+
import * as fs from "fs-extra";
12
import { setLogLevel } from "../log";
23
import { NotionBlock } from "../types";
34
import { standardVideoTransformer } from "./VideoTransformer";
4-
import { blocksToMarkdown } from "./pluginTestRun";
5+
import { blocksToMarkdown, kTemporaryTestDirectory } from "./pluginTestRun";
6+
7+
beforeAll(async () => {
8+
try {
9+
if (await fs.pathExists(kTemporaryTestDirectory)) {
10+
await fs.emptyDir(kTemporaryTestDirectory);
11+
} else {
12+
await fs.mkdirp(kTemporaryTestDirectory);
13+
}
14+
} catch (err) {
15+
console.error("Error in beforeAll:", err);
16+
}
17+
});
18+
19+
afterAll(async () => {
20+
try {
21+
await fs.remove(kTemporaryTestDirectory);
22+
} catch (err) {
23+
console.error("Error in afterAll:", err);
24+
}
25+
});
526

627
test("youtube embedded", async () => {
728
const config = { plugins: [standardVideoTransformer] };
@@ -89,6 +110,9 @@ test("video link, not embedded", async () => {
89110
test("direct upload to to Notion (embedded)", async () => {
90111
setLogLevel("verbose");
91112
const config = { plugins: [standardVideoTransformer] };
113+
114+
const fileName1 = "first_video.mp4";
115+
const fileName2 = "second_video.mp4";
92116
const result = await blocksToMarkdown(config, [
93117
{
94118
object: "block",
@@ -103,13 +127,39 @@ test("direct upload to to Notion (embedded)", async () => {
103127
caption: [],
104128
type: "file",
105129
file: {
106-
url: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/people_fre_motionAsset_p3.mp4?X-Blah-blah",
130+
url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName1}?X-Blah-blah`,
131+
},
132+
},
133+
} as unknown as NotionBlock,
134+
{
135+
object: "block",
136+
id: "12f7db3b-4412-4be9-a3f7-6ac423fee94b",
137+
parent: {
138+
type: "page_id",
139+
page_id: "edaffeb2-ece8-4d44-976f-351e6b5757bb",
140+
},
141+
142+
type: "video",
143+
video: {
144+
caption: [],
145+
type: "file",
146+
file: {
147+
url: `https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/${fileName2}?X-Blah-blah`,
107148
},
108149
},
109150
} as unknown as NotionBlock,
110151
]);
152+
111153
expect(result).toContain(`import ReactPlayer from "react-player";`);
112-
expect(result).toContain(
113-
`<ReactPlayer controls url="https://s3.us-west-2.amazonaws.com/secure.notion-static.com/f6bc4746-011e-2124-86ca-ed4337d70891/people_fre_motionAsset_p3.mp4?X-Blah-blah" />`
114-
);
154+
expect(result).toContain(`import video1 from "./${fileName1}";`);
155+
expect(result).toContain(`import video2 from "./${fileName2}";`);
156+
expect(result).toContain(`<ReactPlayer controls url={video1} />`);
157+
expect(result).toContain(`<ReactPlayer controls url={video2} />`);
158+
159+
// Wait half a second for the files to be written
160+
await new Promise(resolve => setTimeout(resolve, 500));
161+
162+
// We should have actually created files in "tempTestFileDir/"
163+
expect(await fs.pathExists("tempTestFileDir/" + fileName1)).toBe(true);
164+
expect(await fs.pathExists("tempTestFileDir/" + fileName2)).toBe(true);
115165
});

‎src/plugins/VideoTransformer.ts‎

Lines changed: 83 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,96 @@
1+
import * as Path from "path";
12
import { VideoBlockObjectResponse } from "@notionhq/client/build/src/api-endpoints";
3+
import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
24
import { IDocuNotionContext, IPlugin } from "./pluginTypes";
35
import { warning } from "../log";
46
import { NotionBlock } from "../types";
7+
import { writeAsset } from "../assets";
58

69
export const standardVideoTransformer: IPlugin = {
710
name: "video",
811
notionToMarkdownTransforms: [
912
{
1013
type: "video",
11-
getStringFromBlock: (
12-
context: IDocuNotionContext,
13-
block: NotionBlock
14-
): string => {
15-
const video = (block as VideoBlockObjectResponse).video;
16-
let url = "";
17-
switch (video.type) {
18-
case "external":
19-
url = video.external.url;
20-
break;
21-
case "file":
22-
url = video.file.url;
23-
break;
24-
default:
25-
// video.type can only be "external" or "file" as of the writing of this code, so typescript
26-
// isn't happy trying to turn video.type into a string. But this default in our switch is
27-
// just attempting some future-proofing. Thus the strange typing/stringifying below.
28-
warning(
29-
`[standardVideoTransformer] Found Notion "video" block with type ${JSON.stringify(
30-
(video as any).type
31-
)}. The best docu-notion can do for now is ignore it.`
32-
);
33-
return "";
34-
break;
35-
}
36-
37-
context.imports.push(`import ReactPlayer from "react-player";`);
38-
return `<ReactPlayer controls url="${url}" />`;
39-
},
14+
getStringFromBlock: (context: IDocuNotionContext, block: NotionBlock) =>
15+
markdownToMDVideoTransformer(block, context),
4016
},
4117
],
4218
};
19+
20+
async function markdownToMDVideoTransformer(
21+
block: ListBlockChildrenResponseResult,
22+
context: IDocuNotionContext
23+
): Promise<string> {
24+
const videoBlock = block as VideoBlockObjectResponse;
25+
const video = videoBlock.video;
26+
let url = "";
27+
switch (video.type) {
28+
case "external":
29+
url = `"${video.external.url}"`;
30+
break;
31+
case "file":
32+
// The url we get for a Notion-hosted asset expires after an hour, so we have to download it locally.
33+
url = await downloadVideoAndConvertUrl(
34+
context,
35+
video.file.url,
36+
videoBlock.id
37+
);
38+
break;
39+
default:
40+
// video.type can only be "external" or "file" as of the writing of this code, so typescript
41+
// isn't happy trying to turn video.type into a string. But this default in our switch is
42+
// just attempting some future-proofing. Thus the strange typing/stringifying below.
43+
warning(
44+
`[standardVideoTransformer] Found Notion "video" block with type ${JSON.stringify(
45+
(video as any).type
46+
)}. The best docu-notion can do for now is ignore it.`
47+
);
48+
return "";
49+
}
50+
51+
context.imports.push(`import ReactPlayer from "react-player";`);
52+
return `<ReactPlayer controls url=${url} />`;
53+
}
54+
55+
// ENHANCE: One day, we may want to allow for options of where to place the files, how
56+
// to name them, etc. Or we could at least follow the image options.
57+
// But for now, I'm just trying to fix the bug that Notion-hosted videos don't work at all.
58+
async function downloadVideoAndConvertUrl(
59+
context: IDocuNotionContext,
60+
notionVideoUrl: string,
61+
blockId: string
62+
): Promise<string> {
63+
// Get the file name from the url. Ignore query parameters and fragments.
64+
let newFileName = notionVideoUrl.split("?")[0].split("#")[0].split("/").pop();
65+
66+
if (!newFileName) {
67+
// If something went wrong, fall back to the block ID.
68+
// But at least try to get the extension from the url.
69+
const extension = notionVideoUrl
70+
.split("?")[0]
71+
.split("#")[0]
72+
.split(".")
73+
.pop();
74+
newFileName = blockId + (extension ? "." + extension : "");
75+
}
76+
77+
const newPath = Path.posix.join(
78+
context.pageInfo.directoryContainingMarkdown,
79+
newFileName
80+
);
81+
82+
const response = await fetch(notionVideoUrl);
83+
const arrayBuffer = await response.arrayBuffer();
84+
const buffer = Buffer.from(arrayBuffer);
85+
writeAsset(newPath, buffer);
86+
87+
// Add an import statement for the video file.
88+
// Otherwise, the docusaurus build won't include the video file in the build.
89+
const countVideoImports = context.imports.filter(i => {
90+
return /import video\d+/.exec(i);
91+
}).length;
92+
const importName = `video${countVideoImports + 1}`;
93+
context.imports.push(`import ${importName} from "./${newFileName}";`);
94+
95+
return `{${importName}}`;
96+
}

‎src/plugins/pluginTestRun.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import { NotionBlock } from "../types";
1010
import { convertInternalUrl } from "./internalLinks";
1111
import { numberChildrenIfNumberedList } from "../pull";
1212

13+
export const kTemporaryTestDirectory = "tempTestFileDir";
14+
1315
export async function blocksToMarkdown(
1416
config: IDocuNotionConfig,
1517
blocks: NotionBlock[],
@@ -49,7 +51,7 @@ export async function blocksToMarkdown(
4951

5052
//TODO might be needed for some tests, e.g. the image transformer...
5153
pageInfo: {
52-
directoryContainingMarkdown: "not yet",
54+
directoryContainingMarkdown: kTemporaryTestDirectory,
5355
relativeFilePathToFolderContainingPage: "not yet",
5456
slug: "not yet",
5557
},

0 commit comments

Comments
 (0)