diff --git a/proto/api/v1/import_service.proto b/proto/api/v1/import_service.proto new file mode 100644 index 0000000000000..bc62fea1f8452 --- /dev/null +++ b/proto/api/v1/import_service.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package memos.api.v1; + +import "google/api/annotations.proto"; +import "google/api/field_behavior.proto"; + +option go_package = "gen/api/v1"; + +service ImportService { + // ImportMemos imports external note content. + rpc ImportMemos(ImportMemosRequest) returns (ImportResult) { + option (google.api.http) = { + post: "/api/v1/import" + body: "content" + }; + } +} + +message ImportResult {} + +message ImportMemosRequest { + bytes content = 1 [(google.api.field_behavior) = INPUT_ONLY]; +} diff --git a/proto/gen/api/v1/import_service.pb.go b/proto/gen/api/v1/import_service.pb.go new file mode 100644 index 0000000000000..1c72b0fa8a4b7 --- /dev/null +++ b/proto/gen/api/v1/import_service.pb.go @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc (unknown) +// source: api/v1/import_service.proto + +package apiv1 + +import ( + _ "google.golang.org/genproto/googleapis/api/annotations" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ImportResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ImportResult) Reset() { + *x = ImportResult{} + mi := &file_api_v1_import_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ImportResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImportResult) ProtoMessage() {} + +func (x *ImportResult) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_import_service_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImportResult.ProtoReflect.Descriptor instead. +func (*ImportResult) Descriptor() ([]byte, []int) { + return file_api_v1_import_service_proto_rawDescGZIP(), []int{0} +} + +type ImportMemosRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ImportMemosRequest) Reset() { + *x = ImportMemosRequest{} + mi := &file_api_v1_import_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ImportMemosRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImportMemosRequest) ProtoMessage() {} + +func (x *ImportMemosRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_v1_import_service_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImportMemosRequest.ProtoReflect.Descriptor instead. +func (*ImportMemosRequest) Descriptor() ([]byte, []int) { + return file_api_v1_import_service_proto_rawDescGZIP(), []int{1} +} + +func (x *ImportMemosRequest) GetContent() []byte { + if x != nil { + return x.Content + } + return nil +} + +var File_api_v1_import_service_proto protoreflect.FileDescriptor + +var file_api_v1_import_service_proto_rawDesc = string([]byte{ + 0x0a, 0x1b, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, + 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x62, 0x65, 0x68, 0x61, + 0x76, 0x69, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x0e, 0x0a, 0x0c, 0x49, 0x6d, + 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x34, 0x0a, 0x12, 0x49, 0x6d, + 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1e, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0c, 0x42, 0x04, 0xe2, 0x41, 0x01, 0x04, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, + 0x32, 0x7d, 0x0a, 0x0d, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x6c, 0x0a, 0x0b, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x6d, 0x6f, 0x73, + 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, + 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, + 0x31, 0x2e, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x1f, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x19, 0x3a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, + 0x0e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x42, + 0xaa, 0x01, 0x0a, 0x10, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x42, 0x12, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x75, 0x73, 0x65, 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f, + 0x6d, 0x65, 0x6d, 0x6f, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, + 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, + 0x41, 0x58, 0xaa, 0x02, 0x0c, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x2e, 0x41, 0x70, 0x69, 0x2e, 0x56, + 0x31, 0xca, 0x02, 0x0c, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x5c, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, + 0xe2, 0x02, 0x18, 0x4d, 0x65, 0x6d, 0x6f, 0x73, 0x5c, 0x41, 0x70, 0x69, 0x5c, 0x56, 0x31, 0x5c, + 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x0e, 0x4d, 0x65, + 0x6d, 0x6f, 0x73, 0x3a, 0x3a, 0x41, 0x70, 0x69, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_api_v1_import_service_proto_rawDescOnce sync.Once + file_api_v1_import_service_proto_rawDescData []byte +) + +func file_api_v1_import_service_proto_rawDescGZIP() []byte { + file_api_v1_import_service_proto_rawDescOnce.Do(func() { + file_api_v1_import_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_api_v1_import_service_proto_rawDesc), len(file_api_v1_import_service_proto_rawDesc))) + }) + return file_api_v1_import_service_proto_rawDescData +} + +var file_api_v1_import_service_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_api_v1_import_service_proto_goTypes = []any{ + (*ImportResult)(nil), // 0: memos.api.v1.ImportResult + (*ImportMemosRequest)(nil), // 1: memos.api.v1.ImportMemosRequest +} +var file_api_v1_import_service_proto_depIdxs = []int32{ + 1, // 0: memos.api.v1.ImportService.ImportMemos:input_type -> memos.api.v1.ImportMemosRequest + 0, // 1: memos.api.v1.ImportService.ImportMemos:output_type -> memos.api.v1.ImportResult + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_api_v1_import_service_proto_init() } +func file_api_v1_import_service_proto_init() { + if File_api_v1_import_service_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_v1_import_service_proto_rawDesc), len(file_api_v1_import_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_api_v1_import_service_proto_goTypes, + DependencyIndexes: file_api_v1_import_service_proto_depIdxs, + MessageInfos: file_api_v1_import_service_proto_msgTypes, + }.Build() + File_api_v1_import_service_proto = out.File + file_api_v1_import_service_proto_goTypes = nil + file_api_v1_import_service_proto_depIdxs = nil +} diff --git a/proto/gen/api/v1/import_service.pb.gw.go b/proto/gen/api/v1/import_service.pb.gw.go new file mode 100644 index 0000000000000..68028e0183959 --- /dev/null +++ b/proto/gen/api/v1/import_service.pb.gw.go @@ -0,0 +1,154 @@ +// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. +// source: api/v1/import_service.proto + +/* +Package apiv1 is a reverse proxy. + +It translates gRPC into RESTful JSON APIs. +*/ +package apiv1 + +import ( + "context" + "errors" + "io" + "net/http" + + "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" + "github.com/grpc-ecosystem/grpc-gateway/v2/utilities" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// Suppress "imported and not used" errors +var ( + _ codes.Code + _ io.Reader + _ status.Status + _ = errors.New + _ = runtime.String + _ = utilities.NewDoubleArray + _ = metadata.Join +) + +func request_ImportService_ImportMemos_0(ctx context.Context, marshaler runtime.Marshaler, client ImportServiceClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ImportMemosRequest + metadata runtime.ServerMetadata + ) + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq.Content); err != nil && !errors.Is(err, io.EOF) { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := client.ImportMemos(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err +} + +func local_request_ImportService_ImportMemos_0(ctx context.Context, marshaler runtime.Marshaler, server ImportServiceServer, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var ( + protoReq ImportMemosRequest + metadata runtime.ServerMetadata + ) + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq.Content); err != nil && !errors.Is(err, io.EOF) { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := server.ImportMemos(ctx, &protoReq) + return msg, metadata, err +} + +// RegisterImportServiceHandlerServer registers the http handlers for service ImportService to "mux". +// UnaryRPC :call ImportServiceServer directly. +// StreamingRPC :currently unsupported pending https://github.com/grpc/grpc-go/issues/906. +// Note that using this registration option will cause many gRPC library features to stop working. Consider using RegisterImportServiceHandlerFromEndpoint instead. +// GRPC interceptors will not work for this type of registration. To use interceptors, you must use the "runtime.WithMiddlewares" option in the "runtime.NewServeMux" call. +func RegisterImportServiceHandlerServer(ctx context.Context, mux *runtime.ServeMux, server ImportServiceServer) error { + mux.Handle(http.MethodPost, pattern_ImportService_ImportMemos_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + var stream runtime.ServerTransportStream + ctx = grpc.NewContextWithServerTransportStream(ctx, &stream) + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateIncomingContext(ctx, mux, req, "/memos.api.v1.ImportService/ImportMemos", runtime.WithHTTPPathPattern("/api/v1/import")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := local_request_ImportService_ImportMemos_0(annotatedContext, inboundMarshaler, server, req, pathParams) + md.HeaderMD, md.TrailerMD = metadata.Join(md.HeaderMD, stream.Header()), metadata.Join(md.TrailerMD, stream.Trailer()) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_ImportService_ImportMemos_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + + return nil +} + +// RegisterImportServiceHandlerFromEndpoint is same as RegisterImportServiceHandler but +// automatically dials to "endpoint" and closes the connection when "ctx" gets done. +func RegisterImportServiceHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { + conn, err := grpc.NewClient(endpoint, opts...) + if err != nil { + return err + } + defer func() { + if err != nil { + if cerr := conn.Close(); cerr != nil { + grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) + } + return + } + go func() { + <-ctx.Done() + if cerr := conn.Close(); cerr != nil { + grpclog.Errorf("Failed to close conn to %s: %v", endpoint, cerr) + } + }() + }() + return RegisterImportServiceHandler(ctx, mux, conn) +} + +// RegisterImportServiceHandler registers the http handlers for service ImportService to "mux". +// The handlers forward requests to the grpc endpoint over "conn". +func RegisterImportServiceHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { + return RegisterImportServiceHandlerClient(ctx, mux, NewImportServiceClient(conn)) +} + +// RegisterImportServiceHandlerClient registers the http handlers for service ImportService +// to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "ImportServiceClient". +// Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "ImportServiceClient" +// doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in +// "ImportServiceClient" to call the correct interceptors. This client ignores the HTTP middlewares. +func RegisterImportServiceHandlerClient(ctx context.Context, mux *runtime.ServeMux, client ImportServiceClient) error { + mux.Handle(http.MethodPost, pattern_ImportService_ImportMemos_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + annotatedContext, err := runtime.AnnotateContext(ctx, mux, req, "/memos.api.v1.ImportService/ImportMemos", runtime.WithHTTPPathPattern("/api/v1/import")) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_ImportService_ImportMemos_0(annotatedContext, inboundMarshaler, client, req, pathParams) + annotatedContext = runtime.NewServerMetadataContext(annotatedContext, md) + if err != nil { + runtime.HTTPError(annotatedContext, mux, outboundMarshaler, w, req, err) + return + } + forward_ImportService_ImportMemos_0(annotatedContext, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + }) + return nil +} + +var ( + pattern_ImportService_ImportMemos_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "import"}, "")) +) + +var ( + forward_ImportService_ImportMemos_0 = runtime.ForwardResponseMessage +) diff --git a/proto/gen/api/v1/import_service_grpc.pb.go b/proto/gen/api/v1/import_service_grpc.pb.go new file mode 100644 index 0000000000000..4f14ec79d2400 --- /dev/null +++ b/proto/gen/api/v1/import_service_grpc.pb.go @@ -0,0 +1,123 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: api/v1/import_service.proto + +package apiv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ImportService_ImportMemos_FullMethodName = "/memos.api.v1.ImportService/ImportMemos" +) + +// ImportServiceClient is the client API for ImportService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ImportServiceClient interface { + // ImportMemos returns the workspace profile. + ImportMemos(ctx context.Context, in *ImportMemosRequest, opts ...grpc.CallOption) (*ImportResult, error) +} + +type importServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewImportServiceClient(cc grpc.ClientConnInterface) ImportServiceClient { + return &importServiceClient{cc} +} + +func (c *importServiceClient) ImportMemos(ctx context.Context, in *ImportMemosRequest, opts ...grpc.CallOption) (*ImportResult, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ImportResult) + err := c.cc.Invoke(ctx, ImportService_ImportMemos_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ImportServiceServer is the server API for ImportService service. +// All implementations must embed UnimplementedImportServiceServer +// for forward compatibility. +type ImportServiceServer interface { + // ImportMemos returns the workspace profile. + ImportMemos(context.Context, *ImportMemosRequest) (*ImportResult, error) + mustEmbedUnimplementedImportServiceServer() +} + +// UnimplementedImportServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedImportServiceServer struct{} + +func (UnimplementedImportServiceServer) ImportMemos(context.Context, *ImportMemosRequest) (*ImportResult, error) { + return nil, status.Errorf(codes.Unimplemented, "method ImportMemos not implemented") +} +func (UnimplementedImportServiceServer) mustEmbedUnimplementedImportServiceServer() {} +func (UnimplementedImportServiceServer) testEmbeddedByValue() {} + +// UnsafeImportServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ImportServiceServer will +// result in compilation errors. +type UnsafeImportServiceServer interface { + mustEmbedUnimplementedImportServiceServer() +} + +func RegisterImportServiceServer(s grpc.ServiceRegistrar, srv ImportServiceServer) { + // If the following call pancis, it indicates UnimplementedImportServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ImportService_ServiceDesc, srv) +} + +func _ImportService_ImportMemos_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ImportMemosRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ImportServiceServer).ImportMemos(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ImportService_ImportMemos_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ImportServiceServer).ImportMemos(ctx, req.(*ImportMemosRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ImportService_ServiceDesc is the grpc.ServiceDesc for ImportService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ImportService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "memos.api.v1.ImportService", + HandlerType: (*ImportServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ImportMemos", + Handler: _ImportService_ImportMemos_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "api/v1/import_service.proto", +} diff --git a/proto/gen/apidocs.swagger.yaml b/proto/gen/apidocs.swagger.yaml index 24314f5ec7f45..7f197f3f30f3d 100644 --- a/proto/gen/apidocs.swagger.yaml +++ b/proto/gen/apidocs.swagger.yaml @@ -7,6 +7,7 @@ tags: - name: UserService - name: AuthService - name: IdentityProviderService + - name: ImportService - name: InboxService - name: MarkdownService - name: ResourceService @@ -175,6 +176,28 @@ paths: $ref: '#/definitions/apiv1IdentityProvider' tags: - IdentityProviderService + /api/v1/import: + post: + summary: ImportMemos imports external note content. + operationId: ImportService_ImportMemos + responses: + "200": + description: A successful response. + schema: + $ref: '#/definitions/v1ImportResult' + default: + description: An unexpected error response. + schema: + $ref: '#/definitions/googlerpcStatus' + parameters: + - name: content + in: body + required: true + schema: + type: string + format: byte + tags: + - ImportService /api/v1/inboxes: get: summary: ListInboxes lists inboxes for a user. @@ -2581,6 +2604,8 @@ definitions: type: string url: type: string + v1ImportResult: + type: object v1Inbox: type: object properties: diff --git a/server/importer/importer.go b/server/importer/importer.go new file mode 100644 index 0000000000000..5e14338ca0c1c --- /dev/null +++ b/server/importer/importer.go @@ -0,0 +1,53 @@ +package importer + +import ( + "context" + "errors" + "log/slog" + + "github.com/usememos/memos/store" +) + +type ImportResult struct { + Memos []store.Memo + Resources []store.Resource + + // Maps resource name to memo indices in[ImportResult.Memos]. + FileMemos map[string][]int +} + +// stateless converter +type memoConverter func(context.Context, []byte) (*ImportResult, error) + +var converters = []struct { + name string + fn memoConverter +}{ + {"takeout", takeoutConverter}, +} + +type Importer struct { + store *store.Store +} + +func New(store *store.Store) *Importer { + return &Importer{ + store: store, + } +} + +func (imp *Importer) Convert(ctx context.Context, content []byte) (*ImportResult, error) { + for _, converter := range converters { + slog.DebugContext(ctx, "importing notes", "converter", converter.name) + + res, err := converter.fn(ctx, content) + if err != nil { + slog.ErrorContext(ctx, "import failed", "converter", converter.name, "err", err) + continue + } + + return res, nil + } + + return nil, errors.New("no converter found") +} diff --git a/server/importer/takeout.go b/server/importer/takeout.go new file mode 100644 index 0000000000000..be14326b95cfc --- /dev/null +++ b/server/importer/takeout.go @@ -0,0 +1,288 @@ +package importer + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "mime" + "path" + "strings" + + storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/store" +) + +const keepRoot = "Takeout/Keep" + +type scannerState int + +const ( + scannerRoot scannerState = iota + scannerKeep +) + +type keepScanner struct { + dir fs.FS + state scannerState + + result *ImportResult +} + +func (s *keepScanner) walk() error { + return fs.WalkDir(s.dir, ".", s.scan) +} + +func (s *keepScanner) scan(fp string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("%s: %w", fp, err) + } + + switch s.state { + case scannerRoot: + err = s.handleRoot(fp, d) + case scannerKeep: + err = s.handleKeep(fp, d) + default: + err = fmt.Errorf("unknown state %d", s.state) + } + + if err != nil { + fmt.Printf("[%d] %s: %s\n", s.state, fp, err) + } + + return nil +} + +func (s *keepScanner) handleRoot(fp string, d fs.DirEntry) error { + if d.IsDir() && fp == keepRoot { + s.state = scannerKeep + } + + return nil +} + +func (s *keepScanner) handleKeep(fp string, d fs.DirEntry) error { + switch { + case !strings.HasPrefix(fp, keepRoot+"/"): + s.state = scannerRoot + return nil + case d.IsDir(): + return fmt.Errorf("unexpected directory: %s", fp) + case strings.EqualFold(d.Name(), "Labels.txt"): + return nil + } + + mimeType := mime.TypeByExtension(path.Ext(fp)) + + switch mimeType { + default: + return fmt.Errorf("unsupported file type: %s", path.Ext(fp)) + case "text/html": + return nil + case "image/jpeg", "image/png": + return s.handleFile(fp, s.handleImage) + case "application/json": + return s.handleFile(fp, s.handleNote) + } +} + +func (s *keepScanner) handleFile(fp string, handler func(fs.File) error) error { + f, err := s.dir.Open(fp) + if err != nil { + return fmt.Errorf("open: %w", err) + } + defer f.Close() + + return handler(f) +} + +func (s *keepScanner) handleImage(f fs.File) error { + info, err := f.Stat() + if err != nil { + return fmt.Errorf("stat: %w", err) + } + + name := info.Name() + updatedTs := info.ModTime().Unix() + + res := store.Resource{ + // ID, UID, MemoID + + // CreatorID: + CreatedTs: updatedTs, + UpdatedTs: updatedTs, + + Filename: name, + Type: mime.TypeByExtension(path.Ext(name)), + Size: info.Size(), + // StorageType, Reference, Payload + } + + res.Blob, err = io.ReadAll(f) + if err != nil { + return fmt.Errorf("read: %w", err) + } + + s.result.Resources = append(s.result.Resources, res) + + return nil +} + +type keepNote struct { + Color string `json:"color"` + IsTrashed bool `json:"isTrashed"` + IsPinned bool `json:"isPinned"` + IsArchived bool `json:"isArchived"` + TextContent string `json:"textContent"` + Title string `json:"title"` + UserEditedTimestampUsec int64 `json:"userEditedTimestampUsec"` + CreatedTimestampUsec int64 `json:"createdTimestampUsec"` + TextContentHTML string `json:"textContentHtml"` + + Labels []struct { + Name string `json:"name"` + } `json:"labels"` + + Annotations []struct { + Description string `json:"description"` + Source string `json:"source"` + Title string `json:"title"` + URL string `json:"url"` + } `json:"annotations"` + + Attachments []struct { + FilePath string `json:"filePath"` + Mimetype string `json:"mimetype"` + } `json:"attachments"` + + ListContent []struct { + TextHTML string `json:"textHtml"` + Text string `json:"text"` + IsChecked bool `json:"isChecked"` + } `json:"listContent"` +} + +func (s *keepScanner) handleNote(f fs.File) error { + dec := json.NewDecoder(f) + dec.DisallowUnknownFields() + + var note keepNote + if err := dec.Decode(¬e); err != nil { + return fmt.Errorf("decode: %w", err) + } + + var content strings.Builder + + if note.Title != "" { + content.WriteString("# " + note.Title + "\n\n") + } + + if note.TextContent != "" { + content.WriteString(note.TextContent + "\n\n") + } + + memo := store.Memo{ + // ID, UID, ParentID + + // RowStatus: + // CreatorID: + CreatedTs: note.CreatedTimestampUsec / 1e6, + UpdatedTs: note.UserEditedTimestampUsec / 1e6, + + // Content: + Visibility: store.Private, // There are no privacy settings in keep + Pinned: note.IsPinned, + Payload: new(storepb.MemoPayload), + } + + memo.Payload.Tags = append(memo.Payload.Tags, "keep/imported") + + switch { + case note.IsTrashed: + memo.Payload.Tags = append(memo.Payload.Tags, "keep/trashed") + fallthrough + case note.IsArchived: + memo.RowStatus = store.Archived + default: + memo.RowStatus = store.Normal + } + + for _, label := range note.Labels { + memo.Payload.Tags = append(memo.Payload.Tags, label.Name) + } + + curMemo := len(s.result.Memos) + for _, attachment := range note.Attachments { + refs := s.result.FileMemos[attachment.FilePath] + s.result.FileMemos[attachment.FilePath] = append(refs, curMemo) + } + + memo.Payload.Property.HasLink = len(note.Annotations) > 0 + memo.Payload.Property.HasTaskList = len(note.ListContent) > 0 + + for _, item := range note.Annotations { + content.WriteString("# ") + + if item.Title != "" { + content.WriteString(item.Title + ": ") + } + + if item.URL != "" { + if item.Source != "" { + content.WriteString("[" + item.Source + "]") + } else { + content.WriteString("[" + item.URL + "]") + } + + content.WriteString("(" + item.URL + ")\n") + } else if item.Source != "" { + content.WriteString(item.Source + "\n") + } + + if item.Description != "" { + content.WriteString(item.Description + "\n\n") + } + } + + for _, item := range note.ListContent { + memo.Payload.Property.HasIncompleteTasks = memo.Payload.Property.HasIncompleteTasks || !item.IsChecked + + content.WriteString("- [") + if item.IsChecked { + content.WriteRune('x') + } else { + content.WriteRune(' ') + } + + content.WriteString("] " + item.Text + "\n") + } + + memo.Content = content.String() + + s.result.Memos = append(s.result.Memos, memo) + return nil +} + +func takeoutConverter(ctx context.Context, data []byte) (*ImportResult, error) { + var scanner keepScanner + + buf := bytes.NewReader(data) + + var err error + scanner.dir, err = zip.NewReader(buf, int64(buf.Len())) + if err != nil { + return nil, fmt.Errorf("open zip: %w", err) + } + + err = scanner.walk() + if err != nil { + return nil, fmt.Errorf("read content: %w", err) + } + + return scanner.result, errors.New("not implemented") +} diff --git a/server/router/api/v1/import_service.go b/server/router/api/v1/import_service.go new file mode 100644 index 0000000000000..bde4e26e02245 --- /dev/null +++ b/server/router/api/v1/import_service.go @@ -0,0 +1,114 @@ +package v1 + +import ( + "context" + "encoding/binary" + "fmt" + "log/slog" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/lithammer/shortuuid/v4" + v1pb "github.com/usememos/memos/proto/gen/api/v1" + "github.com/usememos/memos/server/runner/memopayload" + "github.com/usememos/memos/store" +) + +func (s *APIV1Service) ImportMemos(ctx context.Context, req *v1pb.ImportMemosRequest) (*v1pb.ImportResult, error) { + size := binary.Size(req.Content) + if lim := s.uploadSizeLimit(ctx); size > lim { + return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit") + } + + u, err := s.GetCurrentUser(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to get current user: %v", err) + } + + result, err := s.importer.Convert(ctx, req.Content) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to importe content: %v", err) + } + + for i := range result.Memos { + memo := &result.Memos[i] + memo.UID = shortuuid.New() + memo.CreatorID = u.ID + + err := s.importMemo(ctx, memo) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to import memo %s: %v", memo.UID, err) + } + } + + for _, res := range result.Resources { + res.UID = shortuuid.New() + res.CreatorID = u.ID + + refs := result.FileMemos[res.Filename] + if len(refs) == 0 { + err = s.importResource(ctx, nil, &res) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to import resource %s: %v", res.Filename, err) + } + continue + } + + for _, ref := range refs { + res := res // Copy to create new resource for each ref. + + err = s.importResource(ctx, &result.Memos[ref].ID, &res) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to import resource %s: %v", res.Filename, err) + } + } + } + + return &v1pb.ImportResult{}, nil +} + +func (s *APIV1Service) importResource(ctx context.Context, memoID *int32, res *store.Resource) error { + if memoID != nil { + res.MemoID = memoID + } + + if err := SaveResourceBlob(ctx, s.Store, res); err != nil { + return fmt.Errorf("save resource blob: %w", err) + } + + created, err := s.Store.CreateResource(ctx, res) + if err != nil { + return fmt.Errorf("create resource: %v", err) + } + + *res = *created + + return nil +} + +func (s *APIV1Service) importMemo(ctx context.Context, memo *store.Memo) error { + err := memopayload.RebuildMemoPayload(memo) + if err != nil { + return fmt.Errorf("rebuild memo payload: %w", err) + } + + created, err := s.Store.CreateMemo(ctx, memo) + if err != nil { + return err + } + + *memo = *created // Override memo.ID. + + msg, err := s.convertMemoFromStore(ctx, memo) + if err != nil { + return fmt.Errorf("convert memo: %w", err) + } + + // Try to dispatch webhook when memo is created. + if err := s.DispatchMemoCreatedWebhook(ctx, msg); err != nil { + slog.WarnContext(ctx, "Failed to dispatch memo created webhook", "err", err) + } + + return nil +} diff --git a/server/router/api/v1/resource_service.go b/server/router/api/v1/resource_service.go index ec1f609a9ef84..a60e9dcf43997 100644 --- a/server/router/api/v1/resource_service.go +++ b/server/router/api/v1/resource_service.go @@ -57,18 +57,11 @@ func (s *APIV1Service) CreateResource(ctx context.Context, request *v1pb.CreateR Type: request.Resource.Type, } - workspaceStorageSetting, err := s.Store.GetWorkspaceStorageSetting(ctx) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get workspace storage setting: %v", err) - } size := binary.Size(request.Resource.Content) - uploadSizeLimit := int(workspaceStorageSetting.UploadSizeLimitMb) * MebiByte - if uploadSizeLimit == 0 { - uploadSizeLimit = MaxUploadBufferSizeBytes - } - if size > uploadSizeLimit { + if lim := s.uploadSizeLimit(ctx); size > lim { return nil, status.Errorf(codes.InvalidArgument, "file size exceeds the limit") } + create.Size = int64(size) create.Blob = request.Resource.Content if err := SaveResourceBlob(ctx, s.Store, create); err != nil { @@ -168,7 +161,7 @@ func (s *APIV1Service) GetResourceBinary(ctx context.Context, request *v1pb.GetR if request.Thumbnail && util.HasPrefixes(resource.Type, SupportedThumbnailMimeTypes...) { thumbnailBlob, err := s.getOrGenerateThumbnail(resource) if err != nil { - // thumbnail failures are logged as warnings and not cosidered critical failures as + // thumbnail failures are logged as warnings and not considered critical failures as // a resource image can be used in its place. slog.Warn("failed to get resource thumbnail image", slog.Any("error", err)) } else { @@ -315,7 +308,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc defer dst.Close() // Write the blob to the file. - if err := os.WriteFile(osPath, create.Blob, 0644); err != nil { + if err := os.WriteFile(osPath, create.Blob, 0o644); err != nil { return errors.Wrap(err, "Failed to write file") } create.Reference = internalPath @@ -324,7 +317,7 @@ func SaveResourceBlob(ctx context.Context, s *store.Store, create *store.Resourc } else if workspaceStorageSetting.StorageType == storepb.WorkspaceStorageSetting_S3 { s3Config := workspaceStorageSetting.S3Config if s3Config == nil { - return errors.Errorf("No actived external storage found") + return errors.Errorf("No active external storage found") } s3Client, err := s3.NewClient(ctx, s3Config) if err != nil { diff --git a/server/router/api/v1/v1.go b/server/router/api/v1/v1.go index 901d141ecbb1c..27a4f123b53ce 100644 --- a/server/router/api/v1/v1.go +++ b/server/router/api/v1/v1.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc/reflection" v1pb "github.com/usememos/memos/proto/gen/api/v1" + "github.com/usememos/memos/server/importer" "github.com/usememos/memos/server/profile" "github.com/usememos/memos/store" ) @@ -33,22 +34,27 @@ type APIV1Service struct { v1pb.UnimplementedWebhookServiceServer v1pb.UnimplementedMarkdownServiceServer v1pb.UnimplementedIdentityProviderServiceServer + v1pb.UnimplementedImportServiceServer Secret string Profile *profile.Profile Store *store.Store + importer *importer.Importer + grpcServer *grpc.Server } -func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store, grpcServer *grpc.Server) *APIV1Service { +func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store, imp *importer.Importer, grpcServer *grpc.Server) *APIV1Service { grpc.EnableTracing = true apiv1Service := &APIV1Service{ Secret: secret, Profile: profile, Store: store, + importer: imp, grpcServer: grpcServer, } + grpc_health_v1.RegisterHealthServer(grpcServer, apiv1Service) v1pb.RegisterWorkspaceServiceServer(grpcServer, apiv1Service) v1pb.RegisterWorkspaceSettingServiceServer(grpcServer, apiv1Service) @@ -61,6 +67,8 @@ func NewAPIV1Service(secret string, profile *profile.Profile, store *store.Store v1pb.RegisterWebhookServiceServer(grpcServer, apiv1Service) v1pb.RegisterMarkdownServiceServer(grpcServer, apiv1Service) v1pb.RegisterIdentityProviderServiceServer(grpcServer, apiv1Service) + v1pb.RegisterImportServiceServer(grpcServer, apiv1Service) + reflection.Register(grpcServer) return apiv1Service } @@ -110,6 +118,10 @@ func (s *APIV1Service) RegisterGateway(ctx context.Context, echoServer *echo.Ech if err := v1pb.RegisterIdentityProviderServiceHandler(ctx, gwMux, conn); err != nil { return err } + if err := v1pb.RegisterImportServiceHandler(ctx, gwMux, conn); err != nil { + return err + } + gwGroup := echoServer.Group("") gwGroup.Use(middleware.CORS()) handler := echo.WrapHandler(gwMux) diff --git a/server/router/api/v1/workspace_setting_service.go b/server/router/api/v1/workspace_setting_service.go index 626b6de1b604f..9fc730c7667d6 100644 --- a/server/router/api/v1/workspace_setting_service.go +++ b/server/router/api/v1/workspace_setting_service.go @@ -3,6 +3,7 @@ package v1 import ( "context" "fmt" + "log/slog" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -78,6 +79,20 @@ func (s *APIV1Service) SetWorkspaceSetting(ctx context.Context, request *v1pb.Se return convertWorkspaceSettingFromStore(workspaceSetting), nil } +func (s *APIV1Service) uploadSizeLimit(ctx context.Context) int { + wss, err := s.Store.GetWorkspaceStorageSetting(ctx) + if err != nil { + slog.ErrorContext(ctx, "failed to get workspace storage setting", "err", err) + return -1 + } + + if lim := wss.UploadSizeLimitMb; lim != 0 { + return int(lim) * MebiByte + } + + return MaxUploadBufferSizeBytes +} + func convertWorkspaceSettingFromStore(setting *storepb.WorkspaceSetting) *v1pb.WorkspaceSetting { workspaceSetting := &v1pb.WorkspaceSetting{ Name: fmt.Sprintf("%s%s", WorkspaceSettingNamePrefix, setting.Key.String()), diff --git a/server/server.go b/server/server.go index bb481d2347aba..371f6522de9e1 100644 --- a/server/server.go +++ b/server/server.go @@ -18,6 +18,7 @@ import ( "google.golang.org/grpc" storepb "github.com/usememos/memos/proto/gen/store" + "github.com/usememos/memos/server/importer" "github.com/usememos/memos/server/profile" apiv1 "github.com/usememos/memos/server/router/api/v1" "github.com/usememos/memos/server/router/frontend" @@ -83,7 +84,9 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store )) s.grpcServer = grpcServer - apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store, grpcServer) + imp := importer.New(s.Store) + + apiV1Service := apiv1.NewAPIV1Service(s.Secret, profile, store, imp, grpcServer) // Register gRPC gateway as api v1. if err := apiV1Service.RegisterGateway(ctx, echoServer); err != nil { return nil, errors.Wrap(err, "failed to register gRPC gateway")