From 7d5e6b42341a60d89dc0b6068609ab52988806e7 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:38:52 -0400 Subject: [PATCH 1/8] Import portable browser profile data Import bookmarks, recent history, local storage, and selected Web Store extensions alongside cookies. Keep values local until final approval, enforce the API payload bounds in the CLI, and apply extension entitlements before upload. --- cmd/browser_import_managed_auth.go | 39 ++ cmd/browser_import_profile_data.go | 335 +++++++++++++++++ cmd/browser_import_profile_data_test.go | 55 +++ cmd/connector.go | 1 + cmd/profiles_import_local.go | 53 ++- cmd/profiles_import_local_test.go | 13 + go.mod | 2 + go.sum | 22 ++ internal/browserimport/bundle.go | 116 ++++-- internal/browserimport/bundle_test.go | 59 +-- internal/browserimport/chromium.go | 464 ++++++++++++++++++++++++ internal/browserimport/chromium_test.go | 126 +++++++ internal/browserimport/types.go | 55 +++ 13 files changed, 1288 insertions(+), 52 deletions(-) create mode 100644 cmd/browser_import_profile_data.go create mode 100644 cmd/browser_import_profile_data_test.go diff --git a/cmd/browser_import_managed_auth.go b/cmd/browser_import_managed_auth.go index 9a7081b4..93297520 100644 --- a/cmd/browser_import_managed_auth.go +++ b/cmd/browser_import_managed_auth.go @@ -19,6 +19,11 @@ type managedAuthCapacity struct { unlimited bool } +type storedExtensionCapacity struct { + remaining int + unlimited bool +} + type orgLimitsGetter interface { Get(context.Context, ...option.RequestOption) (*kernel.OrgLimits, error) } @@ -57,6 +62,40 @@ func decodeManagedAuthCapacity(raw string) (managedAuthCapacity, error) { return managedAuthCapacity{remaining: max(0, maxConnections-usedConnections)}, nil } +func loadStoredExtensionCapacity(ctx context.Context, limits orgLimitsGetter) (storedExtensionCapacity, error) { + orgLimits, err := limits.Get(ctx) + if err != nil { + return storedExtensionCapacity{}, err + } + return decodeStoredExtensionCapacity(orgLimits.RawJSON()) +} + +func decodeStoredExtensionCapacity(raw string) (storedExtensionCapacity, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &fields); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode organization limits: %w", err) + } + maxRaw, hasMax := fields["max_stored_extensions"] + usedRaw, hasUsed := fields["stored_extensions_used"] + if !hasMax || !hasUsed { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API does not expose stored extension capacity") + } + if string(maxRaw) == "null" { + return storedExtensionCapacity{unlimited: true}, nil + } + var maximum, used int + if err := json.Unmarshal(maxRaw, &maximum); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode max stored extensions: %w", err) + } + if err := json.Unmarshal(usedRaw, &used); err != nil { + return storedExtensionCapacity{}, fmt.Errorf("decode used stored extensions: %w", err) + } + if maximum < 0 || used < 0 { + return storedExtensionCapacity{}, fmt.Errorf("Kernel API returned invalid stored extension capacity") + } + return storedExtensionCapacity{remaining: max(0, maximum-used)}, nil +} + type managedAuthProvisioner interface { Provision(context.Context, string, []passwordmanager.Record) ([]string, error) Existing(context.Context, string, []passwordmanager.Candidate) (map[string]bool, error) diff --git a/cmd/browser_import_profile_data.go b/cmd/browser_import_profile_data.go new file mode 100644 index 00000000..7311dfe1 --- /dev/null +++ b/cmd/browser_import_profile_data.go @@ -0,0 +1,335 @@ +package cmd + +import ( + "context" + "fmt" + "sort" + "time" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/pterm/pterm" +) + +const ( + bookmarksChoice = "Bookmarks" + historyChoice = "History" + storageChoice = "Local storage" + extensionsChoice = "Browser extensions" +) + +type localProfileDataSelection struct { + data localbrowser.ProfileData + bookmarkCount int + historyCount int + history bool + storage bool + storageSites []string + storageBytes int64 +} + +func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, profile localbrowser.Profile, since time.Time, includeHistory, nonInteractive, humanOutput bool) (localProfileDataSelection, error) { + bookmarks, bookmarkCount, bookmarkErr := localbrowser.ExportBookmarks(profile) + historyCount, historyErr := localbrowser.HistoryCount(ctx, profile, since) + storageSites, storageErr := localbrowser.LocalStorageSites(ctx, profile) + extensions, extensionErr := localbrowser.DiscoverExtensions(profile) + extensionCapacity := storedExtensionCapacity{unlimited: true} + extensionCapacityKnown := c.extensionCapacity == nil + if extensionErr == nil && len(extensions) > 0 && c.extensionCapacity != nil { + var capacityErr error + extensionCapacity, capacityErr = c.extensionCapacity(ctx) + extensionCapacityKnown = capacityErr == nil + if capacityErr != nil && humanOutput { + pterm.Warning.Printf("extension capacity could not be checked; Kernel will enforce it during import: %v\n", capacityErr) + } + } + + if humanOutput { + warnUnavailableBrowserData("bookmarks", bookmarkErr) + warnUnavailableBrowserData("history", historyErr) + warnUnavailableBrowserData("local storage", storageErr) + warnUnavailableBrowserData("extensions", extensionErr) + } + + selection := localProfileDataSelection{history: includeHistory} + options := make([]string, 0, 4) + defaults := make([]string, 0, 4) + labels := make(map[string]string, 4) + if bookmarkErr == nil && bookmarkCount > 0 { + labels[bookmarksChoice] = fmt.Sprintf("%s — %d", bookmarksChoice, bookmarkCount) + options = append(options, labels[bookmarksChoice]) + defaults = append(defaults, labels[bookmarksChoice]) + } + if historyErr == nil && historyCount > 0 { + labels[historyChoice] = fmt.Sprintf("%s — %d visits", historyChoice, historyCount) + options = append(options, labels[historyChoice]) + if includeHistory { + defaults = append(defaults, labels[historyChoice]) + } + } + if storageErr == nil && len(storageSites) > 0 { + total := storageSiteBytes(storageSites) + labels[storageChoice] = fmt.Sprintf("%s — %s across %d origins", storageChoice, formatBinaryBytes(total), len(storageSites)) + options = append(options, labels[storageChoice]) + defaults = append(defaults, labels[storageChoice]) + } + if extensionErr == nil && len(extensions) > 0 { + labels[extensionsChoice] = fmt.Sprintf("%s — %d detected (plan limit applies)", extensionsChoice, len(extensions)) + options = append(options, labels[extensionsChoice]) + defaults = append(defaults, labels[extensionsChoice]) + } + + chosen := defaults + var err error + if !nonInteractive && len(options) > 0 { + chosen, err = c.prompter.MultiSelect("browser data", "use Space to exclude a category", "Choose browser data to import", options, defaults) + if err != nil { + return localProfileDataSelection{}, err + } + } + for _, choice := range chosen { + switch choice { + case labels[bookmarksChoice]: + selection.data.Bookmarks = &bookmarks + selection.bookmarkCount = bookmarkCount + case labels[historyChoice]: + selection.history = true + selection.historyCount = historyCount + case labels[storageChoice]: + selection.storage = true + selection.storageBytes = storageSiteBytes(storageSites) + case labels[extensionsChoice]: + selection.data.Extensions = append(selection.data.Extensions, extensions...) + } + } + + if selection.storage { + selection.storageSites, err = c.chooseLocalStorageSites(storageSites, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + selection.storageBytes = selectedStorageSiteBytes(storageSites, selection.storageSites) + } + if len(selection.data.Extensions) > 0 { + selection.data.Extensions, err = c.chooseProfileExtensions(selection.data.Extensions, extensionCapacity, extensionCapacityKnown, nonInteractive) + if err != nil { + return localProfileDataSelection{}, err + } + } + return selection, nil +} + +func warnUnavailableBrowserData(category string, err error) { + if err != nil { + pterm.Warning.Printf("%s could not be read and will be skipped: %v\n", category, err) + } +} + +func (c ProfilesImportLocalCmd) confirmBrowserImport(targetName string, cookies cookieImportSelection, cookieSites []localbrowser.Site, profileData localProfileDataSelection, logins pendingManagedAuth) (bool, error) { + pterm.Println() + pterm.Printf("Ready to import into profile %q\n\n", targetName) + if cookies.all { + pterm.Printf(" All cookies — %d across %d websites\n", selectedCookieCount(cookieSites, cookies.sites), len(cookies.sites)) + } else { + pterm.Printf(" Cookies from %d selected websites\n", len(cookies.sites)) + } + if profileData.bookmarkCount > 0 { + pterm.Printf(" Bookmarks — %d\n", profileData.bookmarkCount) + } + if profileData.history { + pterm.Printf(" History — %d visits\n", profileData.historyCount) + } + if profileData.storage { + pterm.Printf(" Local storage — %s across %d origins\n", formatBinaryBytes(profileData.storageBytes), len(profileData.storageSites)) + } + if count := len(profileData.data.Extensions); count > 0 { + pterm.Printf(" Browser extensions — %d\n", count) + } + loginCount := 0 + for _, provider := range logins.providers { + loginCount += len(provider.candidates) + } + if loginCount > 0 { + pterm.Printf(" Managed Auth connections — %d\n", loginCount) + } + pterm.Println() + return c.prompter.ConfirmDefault("import browser data", "Proceed?", true) +} + +func selectedCookieCount(sites []localbrowser.Site, selected []string) int { + set := make(map[string]struct{}, len(selected)) + for _, site := range selected { + set[site] = struct{}{} + } + total := 0 + for _, site := range sites { + if _, ok := set[site.Domain]; ok { + total += site.CookieCount + } + } + return total +} + +func (c ProfilesImportLocalCmd) chooseLocalStorageSites(sites []localbrowser.StorageSite, nonInteractive bool) ([]string, error) { + all := make([]string, 0, len(sites)) + for _, site := range sites { + all = append(all, site.Origin) + } + if storageSiteBytes(sites) <= localbrowser.MaxPortableStorageSize { + return all, nil + } + if nonInteractive { + return nil, fmt.Errorf("browser local storage exceeds Kernel's 64 MiB import limit; run interactively to choose websites or disable local storage") + } + + sorted := append([]localbrowser.StorageSite(nil), sites...) + sort.SliceStable(sorted, func(left, right int) bool { + if sorted[left].Bytes == sorted[right].Bytes { + return sorted[left].Origin < sorted[right].Origin + } + return sorted[left].Bytes > sorted[right].Bytes + }) + labels := make([]string, 0, len(sorted)) + byLabel := make(map[string]string, len(sorted)) + defaults := make([]string, 0, len(sorted)) + var selectedBytes int64 + for index, site := range sorted { + label := fmt.Sprintf("%d %s — %s", index+1, compactField(site.Origin, 46), formatBinaryBytes(site.Bytes)) + labels = append(labels, label) + byLabel[label] = site.Origin + if selectedBytes+site.Bytes <= localbrowser.MaxPortableStorageSize { + defaults = append(defaults, label) + selectedBytes += site.Bytes + } + } + chosen, err := c.prompter.MultiSelect("local storage", "deselect websites until the selection fits", "Choose local website data to import (64 MiB maximum)", labels, defaults) + if err != nil { + return nil, err + } + result := make([]string, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil +} + +func (c ProfilesImportLocalCmd) chooseProfileExtensions(extensions []localbrowser.Extension, capacity storedExtensionCapacity, capacityKnown, nonInteractive bool) ([]localbrowser.Extension, error) { + maximum := min(20, len(extensions)) + if capacityKnown && !capacity.unlimited { + maximum = min(maximum, capacity.remaining) + } + if nonInteractive { + if len(extensions) > maximum { + return nil, fmt.Errorf("%d extensions were selected, but only %d can be added under the profile and plan limits; run interactively to choose extensions", len(extensions), maximum) + } + return extensions, nil + } + labels := make([]string, 0, len(extensions)) + byLabel := make(map[string]localbrowser.Extension, len(extensions)) + for index, extension := range extensions { + name := extension.Name + if name == "" { + name = "Unnamed extension" + } + label := fmt.Sprintf("%d %s · %s", index+1, compactField(name, 42), extension.ID[len(extension.ID)-6:]) + labels = append(labels, label) + byLabel[label] = extension + } + defaults := labels[:maximum] + for { + prompt := fmt.Sprintf("Choose extensions to reinstall (select up to %d)", maximum) + chosen, err := c.prompter.MultiSelect("browser extensions", "your Kernel plan controls stored extension capacity", prompt, labels, defaults) + if err != nil { + return nil, err + } + if len(chosen) > maximum { + pterm.Warning.Printf("Select at most %d extension%s\n", maximum, pluralSuffix(maximum)) + defaults = chosen + continue + } + result := make([]localbrowser.Extension, 0, len(chosen)) + for _, label := range chosen { + result = append(result, byLabel[label]) + } + return result, nil + } +} + +func buildSelectedProfileData(ctx context.Context, profile localbrowser.Profile, selection localProfileDataSelection, cookies []localbrowser.Cookie, since time.Time) (localbrowser.ProfileData, map[string]int, error) { + data := selection.data + data.Cookies = cookies + counts := make(map[string]int, 5) + if len(cookies) > 0 { + counts["cookies"] = len(cookies) + } + if data.Bookmarks != nil { + counts["bookmarks"] = selection.bookmarkCount + } + if selection.history { + history, err := localbrowser.ExportHistory(ctx, profile, since) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.History = history + counts["history"] = len(history) + } + if selection.storage { + storage, err := localbrowser.ExportLocalStorage(ctx, profile, selection.storageSites) + if err != nil { + return localbrowser.ProfileData{}, nil, err + } + data.Storage = storage + counts["storage"] = len(storage) + } + if len(data.Extensions) > 0 { + counts["extensions"] = len(data.Extensions) + } + return data, counts, nil +} + +func selectedProfileCategories(counts map[string]int) []string { + order := []string{"cookies", "storage", "bookmarks", "history", "extensions"} + result := make([]string, 0, len(counts)) + for _, category := range order { + if count, selected := counts[category]; selected && count > 0 { + result = append(result, category) + } + } + return result +} + +func storageSiteBytes(sites []localbrowser.StorageSite) int64 { + var total int64 + for _, site := range sites { + total += site.Bytes + } + return total +} + +func selectedStorageSiteBytes(sites []localbrowser.StorageSite, selected []string) int64 { + set := make(map[string]struct{}, len(selected)) + for _, origin := range selected { + set[origin] = struct{}{} + } + var total int64 + for _, site := range sites { + if _, ok := set[site.Origin]; ok { + total += site.Bytes + } + } + return total +} + +func importedStorageOriginCount(records []localbrowser.StorageRecord) int { + origins := make(map[string]struct{}) + for _, record := range records { + origins[record.Origin] = struct{}{} + } + return len(origins) +} + +func formatBinaryBytes(bytes int64) string { + if bytes < 1<<20 { + return fmt.Sprintf("%.1f KiB", float64(bytes)/(1<<10)) + } + return fmt.Sprintf("%.1f MiB", float64(bytes)/(1<<20)) +} diff --git a/cmd/browser_import_profile_data_test.go b/cmd/browser_import_profile_data_test.go new file mode 100644 index 00000000..cf58bb60 --- /dev/null +++ b/cmd/browser_import_profile_data_test.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "strings" + "testing" + + localbrowser "github.com/kernel/cli/internal/browserimport" + "github.com/stretchr/testify/require" +) + +func TestLocalStorageSelectionUsesAllSitesWithinLimit(t *testing.T) { + sites := []localbrowser.StorageSite{ + {Origin: "https://example.com", Bytes: 1024}, + {Origin: "https://other.example", Bytes: 2048}, + } + + selected, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.NoError(t, err) + require.Equal(t, []string{"https://example.com", "https://other.example"}, selected) +} + +func TestLocalStorageSelectionRequiresReviewWhenOverLimit(t *testing.T) { + sites := []localbrowser.StorageSite{{Origin: "https://large.example", Bytes: localbrowser.MaxPortableStorageSize + 1}} + + _, err := (ProfilesImportLocalCmd{}).chooseLocalStorageSites(sites, true) + require.ErrorContains(t, err, "run interactively to choose websites") +} + +func TestNonInteractiveExtensionSelectionHonorsCapacity(t *testing.T) { + extensions := []localbrowser.Extension{ + {ID: strings.Repeat("a", 32), Source: "chrome_web_store"}, + {ID: strings.Repeat("b", 32), Source: "chrome_web_store"}, + } + + _, err := (ProfilesImportLocalCmd{}).chooseProfileExtensions(extensions, storedExtensionCapacity{remaining: 1}, true, true) + require.ErrorContains(t, err, "only 1 can be added") +} + +func TestSelectedProfileCategoriesUsePortableApplyOrder(t *testing.T) { + categories := selectedProfileCategories(map[string]int{ + "extensions": 1, + "bookmarks": 2, + "cookies": 3, + "history": 4, + "storage": 5, + }) + + require.Equal(t, []string{"cookies", "storage", "bookmarks", "history", "extensions"}, categories) +} + +func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) { + flag := profilesImportLocalCmd.Flags().Lookup("history") + require.NotNil(t, flag) + require.Equal(t, "true", flag.DefValue) +} diff --git a/cmd/connector.go b/cmd/connector.go index 34d16212..cc66d607 100644 --- a/cmd/connector.go +++ b/cmd/connector.go @@ -59,6 +59,7 @@ func runConnectorOpen(cmd *cobra.Command, args []string) error { Version: metadata.Version, WaitTimeout: 30 * time.Minute, DashboardLaunch: true, + ImportHistory: true, } project, err := validateConnectorProject(cmd, input.ProjectID) if err != nil { diff --git a/cmd/profiles_import_local.go b/cmd/profiles_import_local.go index 474b2468..1b301a45 100644 --- a/cmd/profiles_import_local.go +++ b/cmd/profiles_import_local.go @@ -41,6 +41,7 @@ type ProfilesImportLocalInput struct { PasswordManager string InstallAgentSkills bool DashboardLaunch bool + ImportHistory bool Project *kernel.Project } @@ -51,6 +52,7 @@ type ProfilesImportLocalCmd struct { providers func() []passwordmanager.Provider provisioner managedAuthProvisioner managedAuthCapacity func(context.Context) (managedAuthCapacity, error) + extensionCapacity func(context.Context) (storedExtensionCapacity, error) } type pendingManagedAuth struct { @@ -185,6 +187,11 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI if len(cookieSelection.sites) == 0 { return fmt.Errorf("select at least one website") } + since := c.now().AddDate(0, 0, -in.Days) + profileDataSelection, err := c.chooseLocalProfileData(ctx, profile, since, in.ImportHistory, nonInteractive, humanOutput) + if err != nil { + return err + } pendingLogins := pendingManagedAuth{} if managedAuthImportRequested(in.PasswordManager, nonInteractive) { phaseStarted = time.Now() @@ -196,6 +203,16 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI return err } } + if !nonInteractive { + proceed, err := c.confirmBrowserImport(targetName, cookieSelection, cookieSites, profileDataSelection, pendingLogins) + if err != nil { + return err + } + if !proceed { + pterm.Info.Println("Browser import canceled; no Kernel resources were changed") + return nil + } + } if humanOutput { pterm.Println() if cookieSelection.all { @@ -224,7 +241,12 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI version = "dev" } phaseStarted = time.Now() - bundle, err := localbrowser.BuildCookieBundle(ctx, profile, targetName, version, cookies) + profileData, itemCounts, err := buildSelectedProfileData(ctx, profile, profileDataSelection, cookies, since) + if err != nil { + return err + } + categories := selectedProfileCategories(itemCounts) + bundle, err := localbrowser.BuildProfileBundle(ctx, profile, targetName, version, profileData) timings["bundle"] = time.Since(phaseStarted) if err != nil { return err @@ -247,13 +269,13 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } inventory := localbrowser.Inventory{Sources: []localbrowser.Source{{ ID: profile.ID, Kind: "browser", Name: profile.DisplayName(), Browser: profile.Browser.ID, - DataTypes: []string{"cookies"}, ItemCounts: map[string]int{"cookies": len(cookies)}, + DataTypes: categories, ItemCounts: itemCounts, }}} status, err := client.SubmitInventory(ctx, created.ID, created.HelperToken, inventory) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) } - selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: []string{"cookies"}}}} + selection := localbrowser.Selection{Profiles: []localbrowser.ProfileSelection{{SourceID: profile.ID, TargetName: targetName, Categories: categories}}} status, err = client.SubmitSelection(ctx, created.ID, selection) if err != nil { return browserImportProgressError(created.ID, status.Phase, time.Since(phaseStarted), err) @@ -275,6 +297,18 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI profileID := status.Applied.Profiles[0].ProfileID if humanOutput { pterm.Success.Printf("Imported %d cookies from %d websites\n", len(cookies), importedCookieSites) + if count := itemCounts["bookmarks"]; count > 0 { + pterm.Success.Printf("Imported %d bookmarks\n", count) + } + if count := itemCounts["history"]; count > 0 { + pterm.Success.Printf("Imported %d history entries\n", count) + } + if count := itemCounts["storage"]; count > 0 { + pterm.Success.Printf("Imported %d local storage keys from %d origins\n", count, importedStorageOriginCount(profileData.Storage)) + } + if count := itemCounts["extensions"]; count > 0 { + pterm.Success.Printf("Installed %d browser extensions\n", count) + } } connectionIDs := make([]string, 0) approvedLogins := make([]passwordmanager.Record, 0) @@ -325,7 +359,7 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI } } if in.Output == "json" { - data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": len(cookies), "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") + data, err := json.MarshalIndent(map[string]any{"profile_id": profileID, "profile_name": targetName, "sites": cookieSelection.sites, "cookies_imported": itemCounts["cookies"], "browser_data_imported": itemCounts, "managed_auth_connections": connectionIDs, "agent_skills_installed": installedSkills, "agent_skill_warning": skillWarning, "duration_ms": time.Since(startedAt).Milliseconds(), "timings_ms": durationMilliseconds(timings)}, "", " ") if err != nil { return err } @@ -1405,8 +1439,8 @@ var cuidLikeProfileName = regexp.MustCompile(`^[a-z0-9]{24}$`) var profilesImportLocalCmd = &cobra.Command{ Use: "import-local", - Short: "Import cookies from a local browser", - Long: "Import all cookies or selected websites from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", + Short: "Import a local browser profile", + Long: "Import cookies and selected portable data from a local Google Chrome or Helium profile on macOS into a Kernel browser profile.", Args: cobra.NoArgs, RunE: runProfilesImportLocal, } @@ -1427,6 +1461,7 @@ func init() { profilesImportLocalCmd.Flags().Int("days", 30, "Rank websites used during the last number of days (1-90)") profilesImportLocalCmd.Flags().Duration("wait-timeout", 30*time.Minute, "Maximum time to wait for the import to complete") profilesImportLocalCmd.Flags().BoolP("yes", "y", false, "Import all cookies and use unambiguous defaults without prompting") + profilesImportLocalCmd.Flags().Bool("history", true, "Include browsing history from the selected --days window") profilesImportLocalCmd.Flags().String("password-manager", "", "Password managers to search: bitwarden, 1password, both comma-separated, all, or none") profilesImportLocalCmd.Flags().Bool("install-agent-skills", false, "Install the Kernel Managed Auth skill into detected agent directories") addJSONOutputFlag(profilesImportLocalCmd) @@ -1442,12 +1477,13 @@ func runProfilesImportLocal(cmd *cobra.Command, _ []string) error { skipConfirm, _ := cmd.Flags().GetBool("yes") passwordManager, _ := cmd.Flags().GetString("password-manager") installAgentSkills, _ := cmd.Flags().GetBool("install-agent-skills") + importHistory, _ := cmd.Flags().GetBool("history") output, _ := cmd.Flags().GetString("output") project, _ := cmd.Flags().GetString("project") return runProfilesImportLocalWithInput(cmd, ProfilesImportLocalInput{ BrowserProfile: browserProfile, ProfileName: profileName, Sites: sites, Days: days, SkipConfirm: skipConfirm, Output: output, ProjectID: resolveProjectSelection(project), Version: metadata.Version, - WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, + WaitTimeout: waitTimeout, PasswordManager: passwordManager, InstallAgentSkills: installAgentSkills, ImportHistory: importHistory, }) } @@ -1484,6 +1520,9 @@ func runProfilesImportLocalWithInput(cmd *cobra.Command, input ProfilesImportLoc providers: passwordmanager.Detect, provisioner: kernelManagedAuthProvisioner{credentials: &credentials, connections: &connections}, managedAuthCapacity: func(ctx context.Context) (managedAuthCapacity, error) { return loadManagedAuthCapacity(ctx, &limits) }, + extensionCapacity: func(ctx context.Context) (storedExtensionCapacity, error) { + return loadStoredExtensionCapacity(ctx, &limits) + }, } input.ProjectID = project.ID if input.Version == "" { diff --git a/cmd/profiles_import_local_test.go b/cmd/profiles_import_local_test.go index 9392f917..0a9bd3bc 100644 --- a/cmd/profiles_import_local_test.go +++ b/cmd/profiles_import_local_test.go @@ -384,6 +384,19 @@ func TestDecodeManagedAuthCapacity(t *testing.T) { }) } +func TestDecodeStoredExtensionCapacity(t *testing.T) { + capacity, err := decodeStoredExtensionCapacity(`{"max_stored_extensions":5,"stored_extensions_used":3}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{remaining: 2}, capacity) + + capacity, err = decodeStoredExtensionCapacity(`{"max_stored_extensions":null,"stored_extensions_used":12}`) + require.NoError(t, err) + assert.Equal(t, storedExtensionCapacity{unlimited: true}, capacity) + + _, err = decodeStoredExtensionCapacity(`{"max_auth_connections":10}`) + require.ErrorContains(t, err, "does not expose stored extension capacity") +} + func TestChooseManagedAuthLoginsRejectsExplicitBatchAboveRemainingConnections(t *testing.T) { command := managedAuthTestCommand(func() []passwordmanager.Provider { return []passwordmanager.Provider{fakePasswordManager{candidates: []passwordmanager.Candidate{ diff --git a/go.mod b/go.mod index d1c59e04..0a10affa 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.11.1 + github.com/syndtr/goleveldb v1.0.0 github.com/zalando/go-keyring v0.2.6 golang.org/x/crypto v0.52.0 golang.org/x/net v0.54.0 @@ -40,6 +41,7 @@ require ( github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/gookit/color v1.5.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect diff --git a/go.sum b/go.sum index dcec3001..de4bd159 100644 --- a/go.sum +++ b/go.sum @@ -50,16 +50,22 @@ github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -95,6 +101,11 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -128,6 +139,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -153,6 +166,7 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -161,11 +175,13 @@ golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -201,7 +217,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index eeb24c9f..11981ca4 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -6,12 +6,17 @@ import ( "context" "encoding/json" "fmt" - "sort" "github.com/klauspost/compress/zstd" ) -const maxBundleBytes = 128 << 20 +const ( + maxBundleBytes = 128 << 20 + maxPortableFileBytes = 64 << 20 + maxPortableRecordBytes = 1 << 20 + maxPortableRecords = 100_000 + maxPortableDocumentBytes = 16 << 20 +) type Manifest struct { Version int `json:"version"` @@ -33,37 +38,109 @@ type BundleProfile struct { } type ProfileFiles struct { - Cookies string `json:"cookies,omitempty"` + Cookies string `json:"cookies,omitempty"` + Storage string `json:"storage,omitempty"` + Bookmarks string `json:"bookmarks,omitempty"` + History string `json:"history,omitempty"` + Extensions string `json:"extensions,omitempty"` +} + +type bundleFile struct { + path string + data []byte } -func BuildCookieBundle(ctx context.Context, profile Profile, targetName, version string, cookies []Cookie) ([]byte, error) { - if len(cookies) == 0 { - return nil, fmt.Errorf("no cookies were selected") +func BuildProfileBundle(ctx context.Context, profile Profile, targetName, version string, data ProfileData) ([]byte, error) { + if len(data.Cookies) == 0 && len(data.Storage) == 0 && data.Bookmarks == nil && len(data.History) == 0 && len(data.Extensions) == 0 { + return nil, fmt.Errorf("no browser data was selected") + } + files := ProfileFiles{} + payloads := make([]bundleFile, 0, 5) + addJSON := func(path, label string, value any) error { + encoded, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("encode browser %s: %w", label, err) + } + if len(encoded) > maxPortableDocumentBytes { + return fmt.Errorf("browser %s exceeds the 16 MiB import limit", label) + } + payloads = append(payloads, bundleFile{path: path, data: encoded}) + return nil + } + if len(data.Cookies) > 0 { + files.Cookies = "profiles/" + profile.ID + "/cookies.jsonl" + encoded, err := encodeJSONL("cookies", data.Cookies) + if err != nil { + return nil, fmt.Errorf("encode browser cookies: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Cookies, data: encoded}) + } + if len(data.Storage) > 0 { + files.Storage = "profiles/" + profile.ID + "/storage.jsonl" + encoded, err := encodeJSONL("local storage", data.Storage) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + payloads = append(payloads, bundleFile{path: files.Storage, data: encoded}) + } + if data.Bookmarks != nil { + files.Bookmarks = "profiles/" + profile.ID + "/bookmarks.json" + if err := addJSON(files.Bookmarks, "bookmarks", data.Bookmarks); err != nil { + return nil, err + } + } + if len(data.History) > 0 { + files.History = "profiles/" + profile.ID + "/history.jsonl" + encoded, err := encodeJSONL("history", data.History) + if err != nil { + return nil, fmt.Errorf("encode browser history: %w", err) + } + payloads = append(payloads, bundleFile{path: files.History, data: encoded}) + } + if len(data.Extensions) > 0 { + files.Extensions = "profiles/" + profile.ID + "/extensions.json" + if err := addJSON(files.Extensions, "extensions", data.Extensions); err != nil { + return nil, err + } } - cookiePath := "profiles/" + profile.ID + "/cookies.jsonl" manifest := Manifest{ Version: BundleVersion, Source: BundleSource{OS: "macos", HelperVersion: version}, Profiles: []BundleProfile{{ ID: profile.ID, Browser: profile.Browser.ID, SourceName: profile.DisplayName(), TargetName: targetName, - Files: ProfileFiles{Cookies: cookiePath}, + Files: files, }}, } manifestData, err := json.Marshal(manifest) if err != nil { return nil, fmt.Errorf("encode import manifest: %w", err) } - var cookieData bytes.Buffer - encoder := json.NewEncoder(&cookieData) - for _, cookie := range cookies { - if err := encoder.Encode(cookie); err != nil { - return nil, fmt.Errorf("encode browser cookie: %w", err) + return encodeBundle(ctx, manifestData, payloads) +} + +func encodeJSONL[T any](label string, records []T) ([]byte, error) { + if len(records) > maxPortableRecords { + return nil, fmt.Errorf("browser %s exceeds the %d record import limit", label, maxPortableRecords) + } + var output bytes.Buffer + for _, record := range records { + encoded, err := json.Marshal(record) + if err != nil { + return nil, err + } + if len(encoded)+1 > maxPortableRecordBytes { + return nil, fmt.Errorf("one browser %s record exceeds the 1 MiB import limit", label) } + if output.Len()+len(encoded)+1 > maxPortableFileBytes { + return nil, fmt.Errorf("browser %s exceeds the 64 MiB import limit", label) + } + output.Write(encoded) + output.WriteByte('\n') } - return encodeBundle(ctx, manifestData, map[string][]byte{cookiePath: cookieData.Bytes()}) + return output.Bytes(), nil } -func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) ([]byte, error) { +func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]byte, error) { var output bytes.Buffer zstdWriter, err := zstd.NewWriter(&output, zstd.WithEncoderConcurrency(1)) if err != nil { @@ -83,13 +160,8 @@ func encodeBundle(ctx context.Context, manifest []byte, files map[string][]byte) if err := write("manifest.json", manifest); err != nil { return nil, err } - paths := make([]string, 0, len(files)) - for path := range files { - paths = append(paths, path) - } - sort.Strings(paths) - for _, path := range paths { - if err := write(path, files[path]); err != nil { + for _, file := range files { + if err := write(file.path, file.data); err != nil { return nil, err } } diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index 72bdb6fe..dc8b7330 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -3,44 +3,57 @@ package browserimport import ( "archive/tar" "bytes" - "context" "encoding/json" "io" + "strings" "testing" "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestBuildCookieBundleMatchesServerContract(t *testing.T) { - profile := Profile{ID: "chrome-default-1234", Name: "Personal", Browser: Browser{ID: "chrome", Name: "Google Chrome"}} - bundle, err := BuildCookieBundle(context.Background(), profile, "my-browser", "test", []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret", Secure: true}}) +func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { + profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} + bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ + Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}, + Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, + Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{{Name: "bookmark_bar", Children: []BookmarkNode{{Title: "Kernel", URL: "https://onkernel.com"}}}}}, + Extensions: []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Source: "chrome_web_store"}}, + }) require.NoError(t, err) decoder, err := zstd.NewReader(bytes.NewReader(bundle)) require.NoError(t, err) defer decoder.Close() reader := tar.NewReader(decoder) - header, err := reader.Next() - require.NoError(t, err) - assert.Equal(t, "manifest.json", header.Name) - manifestData, err := io.ReadAll(reader) - require.NoError(t, err) + files := make(map[string][]byte) + for { + header, err := reader.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + files[header.Name], err = io.ReadAll(reader) + require.NoError(t, err) + } var manifest Manifest - require.NoError(t, json.Unmarshal(manifestData, &manifest)) - assert.Equal(t, BundleVersion, manifest.Version) - assert.Equal(t, "my-browser", manifest.Profiles[0].TargetName) - - header, err = reader.Next() - require.NoError(t, err) - assert.Equal(t, "profiles/chrome-default-1234/cookies.jsonl", header.Name) - var cookie Cookie - require.NoError(t, json.NewDecoder(reader).Decode(&cookie)) - assert.Equal(t, "secret", cookie.Value) + require.NoError(t, json.Unmarshal(files["manifest.json"], &manifest)) + require.NotEmpty(t, manifest.Profiles[0].Files.Cookies) + require.NotEmpty(t, manifest.Profiles[0].Files.Storage) + require.NotEmpty(t, manifest.Profiles[0].Files.Bookmarks) + require.Empty(t, manifest.Profiles[0].Files.History) + require.NotEmpty(t, manifest.Profiles[0].Files.Extensions) } -func TestBuildCookieBundleRequiresCookies(t *testing.T) { - _, err := BuildCookieBundle(context.Background(), Profile{}, "profile", "test", nil) - assert.EqualError(t, err, "no cookies were selected") +func TestEncodeJSONLEnforcesPortableRecordLimits(t *testing.T) { + _, err := encodeJSONL("history", make([]HistoryRecord, maxPortableRecords+1)) + require.ErrorContains(t, err, "100000 record import limit") + + _, err = encodeJSONL("local storage", []StorageRecord{{ + Origin: "https://example.com", + Kind: StorageKindLocal, + Key: "large", + Value: strings.Repeat("x", maxPortableRecordBytes), + }}) + require.ErrorContains(t, err, "1 MiB import limit") } diff --git a/internal/browserimport/chromium.go b/internal/browserimport/chromium.go index bffdaa64..cda6ecb0 100644 --- a/internal/browserimport/chromium.go +++ b/internal/browserimport/chromium.go @@ -7,6 +7,7 @@ import ( "crypto/cipher" "crypto/sha1" "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" @@ -18,9 +19,13 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "time" + "unicode/utf16" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/util" "golang.org/x/crypto/pbkdf2" "golang.org/x/net/publicsuffix" ) @@ -29,9 +34,14 @@ const ( maxDocumentBytes = 16 << 20 maxSQLiteOutput = 64 << 20 maxSQLiteBytes = 2 << 30 + maxStorageBytes = MaxPortableStorageSize + maxStorageSource = 512 << 20 + maxStorageRecord = 1 << 20 + maxStorageCount = 100_000 ) var profileIDCharacters = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) +var chromeWebStoreExtensionID = regexp.MustCompile(`^[a-p]{32}$`) func DiscoverMacOSProfiles(home string) ([]Profile, error) { browsers := []Browser{ @@ -244,6 +254,373 @@ func ExportCookies(ctx context.Context, profile Profile, selectedSites []string) return cookies, nil } +func ExportBookmarks(profile Profile) (BookmarkDocument, int, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Bookmarks"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + return BookmarkDocument{}, 0, nil + } + if err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("read browser bookmarks: %w", err) + } + var source struct { + Roots map[string]chromiumBookmarkNode `json:"roots"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return BookmarkDocument{}, 0, fmt.Errorf("decode browser bookmarks: %w", err) + } + document := BookmarkDocument{Roots: make([]BookmarkRoot, 0, 3)} + count := 0 + for _, name := range []string{"bookmark_bar", "other", "synced"} { + root, ok := source.Roots[name] + if !ok { + continue + } + portableName := name + if name == "synced" { + portableName = "mobile" + } + document.Roots = append(document.Roots, BookmarkRoot{Name: portableName, Children: portableBookmarkNodes(root.Children, &count)}) + } + return document, count, nil +} + +type chromiumBookmarkNode struct { + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + DateAdded string `json:"date_added"` + DateLastUsed string `json:"date_last_used"` + Children []chromiumBookmarkNode `json:"children"` +} + +func portableBookmarkNodes(nodes []chromiumBookmarkNode, count *int) []BookmarkNode { + result := make([]BookmarkNode, 0, len(nodes)) + for _, node := range nodes { + if node.Type == "url" { + parsed, err := url.Parse(node.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + *count++ + result = append(result, BookmarkNode{Title: node.Name, URL: node.URL, DateAdded: chromiumTimestamp(node.DateAdded), DateLastUsed: chromiumTimestamp(node.DateLastUsed)}) + continue + } + if node.Type != "folder" { + continue + } + children := portableBookmarkNodes(node.Children, count) + *count++ + result = append(result, BookmarkNode{Title: node.Name, Children: children, DateAdded: chromiumTimestamp(node.DateAdded)}) + } + return result +} + +func chromiumTimestamp(value string) *time.Time { + microseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || microseconds <= 0 { + return nil + } + parsed := chromiumTime(microseconds) + return &parsed +} + +func ExportHistory(ctx context.Context, profile Profile, since time.Time) ([]HistoryRecord, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT urls.url, urls.title, MAX(visits.visit_time) AS last_visit_time, COUNT(*) AS visit_count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%') GROUP BY urls.id ORDER BY last_visit_time DESC LIMIT 100001`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return nil, fmt.Errorf("read browser history: %w", err) + } + var rows []struct { + URL string `json:"url"` + Title string `json:"title"` + VisitedAt int64 `json:"last_visit_time"` + Count int `json:"visit_count"` + } + if len(bytes.TrimSpace(payload)) != 0 { + if err := json.Unmarshal(payload, &rows); err != nil { + return nil, fmt.Errorf("decode browser history: %w", err) + } + } + if len(rows) > 100000 { + return nil, fmt.Errorf("browser history exceeds the 100000-record import limit; use fewer days") + } + records := make([]HistoryRecord, 0, len(rows)) + for _, row := range rows { + parsed, err := url.Parse(row.URL) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" { + continue + } + records = append(records, HistoryRecord{URL: row.URL, Title: row.Title, VisitedAt: chromiumTime(row.VisitedAt), VisitCount: max(1, row.Count)}) + } + return records, nil +} + +func HistoryCount(ctx context.Context, profile Profile, since time.Time) (int, error) { + const chromeEpochMicros = int64(11_644_473_600_000_000) + cutoff := since.UnixMicro() + chromeEpochMicros + query := fmt.Sprintf(`SELECT COUNT(*) AS count FROM visits JOIN urls ON urls.id = visits.url WHERE visits.visit_time >= %d AND (urls.url LIKE 'http://%%' OR urls.url LIKE 'https://%%')`, cutoff) + payload, err := sqliteSnapshotJSON(ctx, filepath.Join(profile.Path, "History"), query) + if err != nil { + return 0, fmt.Errorf("count browser history: %w", err) + } + var rows []struct { + Count int `json:"count"` + } + if err := json.Unmarshal(payload, &rows); err != nil { + return 0, fmt.Errorf("decode browser history count: %w", err) + } + if len(rows) != 1 || rows[0].Count < 0 { + return 0, fmt.Errorf("browser history count is invalid") + } + return rows[0].Count, nil +} + +func LocalStorageSites(ctx context.Context, profile Profile) ([]StorageSite, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + metadataBytes := make(map[string]int64) + recordBytes := make(map[string]int64) + iterator := db.NewIterator(nil, nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key() + if bytes.HasPrefix(key, []byte("META:")) { + origin, ok := portableStorageOrigin(string(key[len("META:"):])) + if !ok { + continue + } + size, err := localStorageMetadataSize(iterator.Value()) + if err != nil { + return nil, fmt.Errorf("decode local storage metadata for %s: %w", origin, err) + } + metadataBytes[origin] = size + continue + } + if len(key) < 2 || key[0] != '_' { + continue + } + separator := bytes.IndexByte(key[1:], 0) + if separator < 1 { + continue + } + origin, ok := portableStorageOrigin(string(key[1 : 1+separator])) + if ok { + recordBytes[origin] += int64(len(key[2+separator:]) + len(iterator.Value())) + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage metadata: %w", err) + } + sites := make([]StorageSite, 0, len(metadataBytes)+len(recordBytes)) + for origin, bytes := range recordBytes { + if metadataBytes[origin] > bytes { + bytes = metadataBytes[origin] + } + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + delete(metadataBytes, origin) + } + for origin, bytes := range metadataBytes { + sites = append(sites, StorageSite{Origin: origin, Bytes: bytes}) + } + sort.Slice(sites, func(left, right int) bool { return sites[left].Origin < sites[right].Origin }) + return sites, nil +} + +func ExportLocalStorage(ctx context.Context, profile Profile, selectedOrigins []string) ([]StorageRecord, error) { + database, cleanup, err := levelDBSnapshot(ctx, filepath.Join(profile.Path, "Local Storage", "leveldb")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("snapshot browser local storage: %w", err) + } + defer cleanup() + + db, err := leveldb.OpenFile(database, nil) + if err != nil { + return nil, fmt.Errorf("open browser local storage: %w", err) + } + defer db.Close() + + selected := make(map[string]struct{}, len(selectedOrigins)) + for _, origin := range selectedOrigins { + selected[origin] = struct{}{} + } + records := make([]StorageRecord, 0) + encodedBytes := 0 + iterator := db.NewIterator(util.BytesPrefix([]byte("_")), nil) + defer iterator.Release() + for iterator.Next() { + key := iterator.Key()[1:] + separator := bytes.IndexByte(key, 0) + if separator < 1 || separator == len(key)-1 { + continue + } + origin, ok := portableStorageOrigin(string(key[:separator])) + if !ok { + continue + } + if len(selected) > 0 { + if _, ok := selected[origin]; !ok { + continue + } + } + scriptKey, err := decodeChromiumStorageString(key[separator+1:]) + if err != nil || scriptKey == "" { + continue + } + value, err := decodeChromiumStorageString(iterator.Value()) + if err != nil { + continue + } + record := StorageRecord{Origin: origin, Kind: StorageKindLocal, Key: scriptKey, Value: value} + encoded, err := json.Marshal(record) + if err != nil { + return nil, fmt.Errorf("encode browser local storage: %w", err) + } + if len(encoded)+1 > maxStorageRecord { + return nil, fmt.Errorf("local storage key %q for %s exceeds the 1 MiB record limit", scriptKey, origin) + } + encodedBytes += len(encoded) + 1 + if encodedBytes > maxStorageBytes { + return nil, fmt.Errorf("browser local storage exceeds the 64 MiB import limit; choose fewer websites") + } + records = append(records, record) + if len(records) > maxStorageCount { + return nil, fmt.Errorf("browser local storage exceeds the 100000-record import limit; choose fewer websites") + } + } + if err := iterator.Error(); err != nil { + return nil, fmt.Errorf("read browser local storage: %w", err) + } + sort.Slice(records, func(left, right int) bool { + if records[left].Origin == records[right].Origin { + return records[left].Key < records[right].Key + } + return records[left].Origin < records[right].Origin + }) + return records, nil +} + +func portableStorageOrigin(raw string) (string, bool) { + parsed, err := url.Parse(raw) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.User != nil || parsed.Host == "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", false + } + origin := parsed.Scheme + "://" + parsed.Host + if raw != origin && raw != origin+"/" { + return "", false + } + return origin, true +} + +func decodeChromiumStorageString(raw []byte) (string, error) { + if len(raw) == 0 { + return "", fmt.Errorf("string is missing its encoding prefix") + } + switch raw[0] { + case 0: + if len(raw[1:])%2 != 0 { + return "", fmt.Errorf("UTF-16 string has an odd byte length") + } + units := make([]uint16, len(raw[1:])/2) + for index := range units { + units[index] = binary.LittleEndian.Uint16(raw[1+index*2:]) + } + return string(utf16.Decode(units)), nil + case 1: + runes := make([]rune, len(raw)-1) + for index, value := range raw[1:] { + runes[index] = rune(value) + } + return string(runes), nil + default: + return "", fmt.Errorf("unsupported string encoding prefix %d", raw[0]) + } +} + +func localStorageMetadataSize(raw []byte) (int64, error) { + for len(raw) > 0 { + tag, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf tag") + } + raw = raw[read:] + field, wire := tag>>3, tag&7 + if wire != 0 { + return 0, fmt.Errorf("unsupported protobuf wire type %d", wire) + } + value, read := binary.Uvarint(raw) + if read <= 0 { + return 0, fmt.Errorf("invalid protobuf value") + } + raw = raw[read:] + if field == 2 { + if value > uint64(^uint64(0)>>1) { + return 0, fmt.Errorf("size exceeds int64") + } + return int64(value), nil + } + } + return 0, fmt.Errorf("size field is missing") +} + +func DiscoverExtensions(profile Profile) ([]Extension, error) { + payload, err := readFileBounded(filepath.Join(profile.Path, "Secure Preferences"), maxDocumentBytes) + if errors.Is(err, os.ErrNotExist) { + payload, err = readFileBounded(filepath.Join(profile.Path, "Preferences"), maxDocumentBytes) + } + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read browser extensions: %w", err) + } + var source struct { + Extensions struct { + Settings map[string]struct { + State *int `json:"state"` + FromWebStore bool `json:"from_webstore"` + Manifest struct { + Name string `json:"name"` + } `json:"manifest"` + } `json:"settings"` + } `json:"extensions"` + } + if err := json.Unmarshal(payload, &source); err != nil { + return nil, fmt.Errorf("decode browser extensions: %w", err) + } + result := make([]Extension, 0) + for id, setting := range source.Extensions.Settings { + if (setting.State != nil && *setting.State != 1) || !setting.FromWebStore || !chromeWebStoreExtensionID.MatchString(id) { + continue + } + result = append(result, Extension{ID: id, Name: setting.Manifest.Name, Source: "chrome_web_store"}) + } + sort.Slice(result, func(i, j int) bool { + if result[i].Name == result[j].Name { + return result[i].ID < result[j].ID + } + return result[i].Name < result[j].Name + }) + return result, nil +} + // CookieSites returns every website with importable cookies, ranked by recent // browser use. It reads cookie metadata only; values are decrypted by // ExportCookies after the user chooses what to import. @@ -374,6 +751,93 @@ type fileFingerprint struct { modified time.Time } +type directoryFingerprint struct { + name string + size int64 + modified time.Time +} + +func levelDBSnapshot(ctx context.Context, sourceDirectory string) (string, func(), error) { + if _, err := os.Stat(sourceDirectory); err != nil { + return "", nil, err + } + for attempt := 0; attempt < 3; attempt++ { + before, err := fingerprintLevelDB(sourceDirectory) + if err != nil { + return "", nil, err + } + var total int64 + for _, file := range before { + total += file.size + } + if total > maxStorageSource { + return "", nil, fmt.Errorf("browser local storage source exceeds 512 MiB") + } + root, err := os.MkdirTemp("", "kernel-browser-import-leveldb-") + if err != nil { + return "", nil, err + } + destination := filepath.Join(root, "leveldb") + if err := os.Mkdir(destination, 0o700); err != nil { + _ = os.RemoveAll(root) + return "", nil, err + } + changed := false + for _, file := range before { + err := copyFileBounded(ctx, filepath.Join(sourceDirectory, file.name), filepath.Join(destination, file.name), file.size) + if errors.Is(err, os.ErrNotExist) || errors.Is(err, errFileGrew) { + changed = true + break + } + if err != nil { + _ = os.RemoveAll(root) + return "", nil, fmt.Errorf("copy local storage %s: %w", file.name, err) + } + } + after, err := fingerprintLevelDB(sourceDirectory) + if !changed && err == nil && directoryFingerprintsEqual(before, after) { + return destination, func() { _ = os.RemoveAll(root) }, nil + } + _ = os.RemoveAll(root) + } + return "", nil, fmt.Errorf("browser local storage changed while it was being read; try again") +} + +func fingerprintLevelDB(directory string) ([]directoryFingerprint, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + result := make([]directoryFingerprint, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || entry.Name() == "LOCK" || entry.Name() == "LOG" || entry.Name() == "LOG.old" { + continue + } + info, err := entry.Info() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + continue + } + result = append(result, directoryFingerprint{name: entry.Name(), size: info.Size(), modified: info.ModTime()}) + } + sort.Slice(result, func(left, right int) bool { return result[left].name < result[right].name }) + return result, nil +} + +func directoryFingerprintsEqual(left, right []directoryFingerprint) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if left[index] != right[index] { + return false + } + } + return true +} + func sqliteSnapshot(ctx context.Context, databasePath string) (string, func(), error) { if !fileExists(databasePath) { return "", nil, fmt.Errorf("browser database %q was not found", filepath.Base(databasePath)) diff --git a/internal/browserimport/chromium_test.go b/internal/browserimport/chromium_test.go index 188046ac..baa1ee84 100644 --- a/internal/browserimport/chromium_test.go +++ b/internal/browserimport/chromium_test.go @@ -6,6 +6,7 @@ import ( "crypto/aes" "crypto/cipher" "crypto/sha256" + "encoding/binary" "encoding/json" "fmt" "os" @@ -13,9 +14,11 @@ import ( "path/filepath" "testing" "time" + "unicode/utf16" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/syndtr/goleveldb/leveldb" ) func TestDiscoverMacOSProfilesFindsChromeAndHelium(t *testing.T) { @@ -229,6 +232,92 @@ INSERT INTO cookies VALUES assert.Equal(t, []string{"github", "google"}, []string{cookies[0].Name, cookies[1].Name}) } +func TestExportBookmarksNormalizesPortableRoots(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{"roots":{"bookmark_bar":{"children":[{"type":"url","name":"Kernel","url":"https://onkernel.com","date_added":"13400000000000000"},{"type":"url","name":"Local","url":"chrome://settings"}]},"other":{"children":[{"type":"folder","name":"Work","children":[{"type":"url","name":"GitHub","url":"https://github.com"}]}]}}}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Bookmarks"), []byte(payload), 0o600)) + + document, count, err := ExportBookmarks(profile) + require.NoError(t, err) + require.Equal(t, 3, count) + require.Len(t, document.Roots, 2) + require.Equal(t, "https://onkernel.com", document.Roots[0].Children[0].URL) + require.Len(t, document.Roots[0].Children, 1, "non-http bookmarks must not leave the laptop") +} + +func TestExportHistoryUsesRequestedWindow(t *testing.T) { + profile := sqliteProfileFixture(t) + database := filepath.Join(profile.Path, "History") + runSQLite(t, database, ` +CREATE TABLE urls (id INTEGER PRIMARY KEY, url TEXT, title TEXT, last_visit_time INTEGER, visit_count INTEGER); +CREATE TABLE visits (id INTEGER PRIMARY KEY, url INTEGER, visit_time INTEGER); +INSERT INTO urls VALUES + (1, 'https://recent.example', 'Recent', 13400000000000000, 4), + (2, 'chrome://settings', 'Private', 13400000001000000, 2), + (3, 'https://old.example', 'Old', 12000000000000000, 1); +INSERT INTO visits VALUES + (1, 1, 13400000000000000), + (2, 1, 13400000000000001), + (3, 2, 13400000001000000), + (4, 3, 12000000000000000); +`) + + records, err := ExportHistory(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, "https://recent.example", records[0].URL) + require.Equal(t, 2, records[0].VisitCount) + count, err := HistoryCount(t.Context(), profile, time.UnixMicro(13400000000000000-11_644_473_600_000_000)) + require.NoError(t, err) + require.Equal(t, 2, count) +} + +func TestLocalStorageSitesAndExportUseLivePortableRecords(t *testing.T) { + profile := sqliteProfileFixture(t) + databasePath := filepath.Join(profile.Path, "Local Storage", "leveldb") + require.NoError(t, os.MkdirAll(filepath.Dir(databasePath), 0o755)) + database, err := leveldb.OpenFile(databasePath, nil) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, database.Close()) }) + require.NoError(t, database.Put([]byte("META:https://example.com"), localStorageMetadataFixture(1234), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://example.com", "theme"), chromiumStorageStringFixture("dark"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("https://other.example", "emoji"), chromiumStorageStringFixture("hello 世界"), nil)) + require.NoError(t, database.Put(localStorageRecordKeyFixture("chrome-extension://abcdefghijklmnopabcdefghijklmnop", "private"), chromiumStorageStringFixture("skip"), nil)) + sites, err := LocalStorageSites(t.Context(), profile) + require.NoError(t, err) + require.Len(t, sites, 2) + require.Equal(t, StorageSite{Origin: "https://example.com", Bytes: 1234}, sites[0]) + require.Equal(t, "https://other.example", sites[1].Origin) + require.Positive(t, sites[1].Bytes) + + records, err := ExportLocalStorage(t.Context(), profile, []string{"https://example.com"}) + require.NoError(t, err) + require.Equal(t, []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, records) + + records, err = ExportLocalStorage(t.Context(), profile, nil) + require.NoError(t, err) + require.Len(t, records, 2) + require.Equal(t, "hello 世界", records[1].Value) +} + +func TestDiscoverExtensionsUsesPortableAllowlist(t *testing.T) { + profile := sqliteProfileFixture(t) + payload := `{ + "browser":{"theme":{"color_scheme2":2},"show_home_button":true}, + "extensions":{"settings":{ + "abcdefghijklmnopabcdefghijklmnop":{"from_webstore":true,"manifest":{"name":"Eligible"}}, + "bcdefghijklmnopabcdefghijklmnopa":{"state":0,"from_webstore":true,"manifest":{"name":"Disabled"}}, + "cdefghijklmnopabcdefghijklmnopab":{"state":1,"from_webstore":false,"manifest":{"name":"Local"}} + }} +}` + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Preferences"), []byte(payload), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Secure Preferences"), []byte(payload), 0o600)) + + extensions, err := DiscoverExtensions(profile) + require.NoError(t, err) + require.Equal(t, []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Name: "Eligible", Source: "chrome_web_store"}}, extensions) +} + func TestSelectedCookieFilterEscapesInput(t *testing.T) { cte, clause := selectedCookieFilter([]string{"example.com' OR 1=1 --"}, false) assert.Contains(t, cte, "example.com'' or 1=1 --") @@ -314,3 +403,40 @@ func bytesRepeat(value byte, count int) []byte { } return result } + +func localStorageRecordKeyFixture(origin, key string) []byte { + result := append([]byte("_"+origin+"\x00"), chromiumStorageStringFixture(key)...) + return result +} + +func chromiumStorageStringFixture(value string) []byte { + runes := []rune(value) + latin1 := true + for _, value := range runes { + if value > 255 { + latin1 = false + break + } + } + if latin1 { + result := make([]byte, 1, len(runes)+1) + result[0] = 1 + for _, value := range runes { + result = append(result, byte(value)) + } + return result + } + units := utf16.Encode(runes) + result := make([]byte, 1+len(units)*2) + for index, value := range units { + binary.LittleEndian.PutUint16(result[1+index*2:], value) + } + return result +} + +func localStorageMetadataFixture(size uint64) []byte { + result := []byte{8, 1, 16} + buffer := make([]byte, binary.MaxVarintLen64) + written := binary.PutUvarint(buffer, size) + return append(result, buffer[:written]...) +} diff --git a/internal/browserimport/types.go b/internal/browserimport/types.go index aab39361..cbce9af2 100644 --- a/internal/browserimport/types.go +++ b/internal/browserimport/types.go @@ -46,6 +46,61 @@ type Cookie struct { SameSite string `json:"same_site,omitempty"` } +type BookmarkDocument struct { + Roots []BookmarkRoot `json:"roots"` +} + +type BookmarkRoot struct { + Name string `json:"name"` + Children []BookmarkNode `json:"children"` +} + +type BookmarkNode struct { + Title string `json:"title"` + URL string `json:"url,omitempty"` + Children []BookmarkNode `json:"children"` + DateAdded *time.Time `json:"date_added,omitempty"` + DateLastUsed *time.Time `json:"date_last_used,omitempty"` +} + +type HistoryRecord struct { + URL string `json:"url"` + Title string `json:"title,omitempty"` + VisitedAt time.Time `json:"visited_at"` + VisitCount int `json:"visit_count,omitempty"` +} + +const ( + StorageKindLocal = "local_storage" + MaxPortableStorageSize = 64 << 20 +) + +type StorageRecord struct { + Origin string `json:"origin"` + Kind string `json:"kind"` + Key string `json:"key"` + Value string `json:"value"` +} + +type StorageSite struct { + Origin string + Bytes int64 +} + +type Extension struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Source string `json:"source"` +} + +type ProfileData struct { + Cookies []Cookie + Storage []StorageRecord + Bookmarks *BookmarkDocument + History []HistoryRecord + Extensions []Extension +} + type Source struct { ID string `json:"id"` Kind string `json:"kind"` From 2e146fe32fcb3da56531502bd57a12a0f5845a20 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:38:22 -0400 Subject: [PATCH 2/8] Preserve local connector environment --- internal/connector/connector.go | 2 +- internal/connector/connector_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 38e7836b..172d0335 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -226,7 +226,7 @@ set kernelExecutable to "` + appleScriptString(executable) + `" set commandText to quoted form of kernelExecutable & " connector open " & quoted form of incomingURL set scriptPath to «event sysoexec» "/usr/bin/mktemp /tmp/kernel-connector.XXXXXX" set scriptFile to «event rdwropen» POSIX file scriptPath with «class perm» -«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile +«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile «event rdwrclos» scriptFile «event sysoexec» "/bin/chmod 700 " & quoted form of scriptPath «event sysoexec» "/usr/bin/open -a Terminal " & quoted form of scriptPath diff --git a/internal/connector/connector_test.go b/internal/connector/connector_test.go index f810caba..f626d3cf 100644 --- a/internal/connector/connector_test.go +++ b/internal/connector/connector_test.go @@ -134,6 +134,8 @@ func TestMacOSAppleScriptShellQuotesURLAtRuntime(t *testing.T) { assert.Contains(t, script, `/usr/bin/open -a Terminal`) assert.Contains(t, script, `/bin/zsh -lic`) assert.Contains(t, script, `/usr/bin/mktemp /tmp/kernel-connector.XXXXXX`) + assert.Contains(t, script, `/bin/launchctl getenv`) + assert.Contains(t, script, `KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL`) assert.NotContains(t, script, `XXXXXX.command`) assert.Contains(t, script, `Kernel CLI was removed`) assert.Contains(t, script, `«event GURLGURL»`) From 9d84ccd158e7b87d7285d7351daf3537d409f404 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:41:29 -0400 Subject: [PATCH 3/8] Apply connector overrides after shell startup --- internal/connector/connector.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 172d0335..c0810500 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -223,10 +223,10 @@ func bundleIdentifier(contents []byte) (string, error) { func macOSAppleScript(executable string) string { return `on «event GURLGURL» incomingURL set kernelExecutable to "` + appleScriptString(executable) + `" -set commandText to quoted form of kernelExecutable & " connector open " & quoted form of incomingURL +set commandText to "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done; exec " & quoted form of kernelExecutable & " connector open " & quoted form of incomingURL set scriptPath to «event sysoexec» "/usr/bin/mktemp /tmp/kernel-connector.XXXXXX" set scriptFile to «event rdwropen» POSIX file scriptPath with «class perm» -«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "for variable in KERNEL_BASE_URL KERNEL_API_KEY KERNEL_AUTH_BASE_URL; do value=$(/bin/launchctl getenv \"$variable\"); if [[ -n \"$value\" ]]; then export \"$variable=$value\"; fi; done" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile +«event rdwrwrit» "#!/bin/zsh" & linefeed & "rm -f " & quoted form of scriptPath & linefeed & "if [[ ! -x " & quoted form of kernelExecutable & " ]]; then echo 'Kernel CLI was removed. Reinstall it with: brew install kernel/tap/kernel'; read -k 1 '?Press any key to close'; exit 1; fi" & linefeed & "exec /bin/zsh -lic " & quoted form of commandText & linefeed given «class refn»:scriptFile «event rdwrclos» scriptFile «event sysoexec» "/bin/chmod 700 " & quoted form of scriptPath «event sysoexec» "/usr/bin/open -a Terminal " & quoted form of scriptPath From 0ea5d416d5548be8bd5e3b0acd1e4841803744d3 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:28:12 -0400 Subject: [PATCH 4/8] Remove extensions from browser import MVP --- cmd/browser_import_managed_auth.go | 39 ----------- cmd/browser_import_profile_data.go | 89 ++----------------------- cmd/browser_import_profile_data_test.go | 22 ++---- cmd/profiles_import_local.go | 7 -- cmd/profiles_import_local_test.go | 13 ---- internal/browserimport/bundle.go | 17 ++--- internal/browserimport/bundle_test.go | 8 +-- internal/browserimport/chromium.go | 42 ------------ internal/browserimport/chromium_test.go | 18 ----- internal/browserimport/types.go | 16 ++--- 10 files changed, 24 insertions(+), 247 deletions(-) diff --git a/cmd/browser_import_managed_auth.go b/cmd/browser_import_managed_auth.go index 93297520..9a7081b4 100644 --- a/cmd/browser_import_managed_auth.go +++ b/cmd/browser_import_managed_auth.go @@ -19,11 +19,6 @@ type managedAuthCapacity struct { unlimited bool } -type storedExtensionCapacity struct { - remaining int - unlimited bool -} - type orgLimitsGetter interface { Get(context.Context, ...option.RequestOption) (*kernel.OrgLimits, error) } @@ -62,40 +57,6 @@ func decodeManagedAuthCapacity(raw string) (managedAuthCapacity, error) { return managedAuthCapacity{remaining: max(0, maxConnections-usedConnections)}, nil } -func loadStoredExtensionCapacity(ctx context.Context, limits orgLimitsGetter) (storedExtensionCapacity, error) { - orgLimits, err := limits.Get(ctx) - if err != nil { - return storedExtensionCapacity{}, err - } - return decodeStoredExtensionCapacity(orgLimits.RawJSON()) -} - -func decodeStoredExtensionCapacity(raw string) (storedExtensionCapacity, error) { - var fields map[string]json.RawMessage - if err := json.Unmarshal([]byte(raw), &fields); err != nil { - return storedExtensionCapacity{}, fmt.Errorf("decode organization limits: %w", err) - } - maxRaw, hasMax := fields["max_stored_extensions"] - usedRaw, hasUsed := fields["stored_extensions_used"] - if !hasMax || !hasUsed { - return storedExtensionCapacity{}, fmt.Errorf("Kernel API does not expose stored extension capacity") - } - if string(maxRaw) == "null" { - return storedExtensionCapacity{unlimited: true}, nil - } - var maximum, used int - if err := json.Unmarshal(maxRaw, &maximum); err != nil { - return storedExtensionCapacity{}, fmt.Errorf("decode max stored extensions: %w", err) - } - if err := json.Unmarshal(usedRaw, &used); err != nil { - return storedExtensionCapacity{}, fmt.Errorf("decode used stored extensions: %w", err) - } - if maximum < 0 || used < 0 { - return storedExtensionCapacity{}, fmt.Errorf("Kernel API returned invalid stored extension capacity") - } - return storedExtensionCapacity{remaining: max(0, maximum-used)}, nil -} - type managedAuthProvisioner interface { Provision(context.Context, string, []passwordmanager.Record) ([]string, error) Existing(context.Context, string, []passwordmanager.Candidate) (map[string]bool, error) diff --git a/cmd/browser_import_profile_data.go b/cmd/browser_import_profile_data.go index 7311dfe1..20344a59 100644 --- a/cmd/browser_import_profile_data.go +++ b/cmd/browser_import_profile_data.go @@ -11,10 +11,9 @@ import ( ) const ( - bookmarksChoice = "Bookmarks" - historyChoice = "History" - storageChoice = "Local storage" - extensionsChoice = "Browser extensions" + bookmarksChoice = "Bookmarks" + historyChoice = "History" + storageChoice = "Local storage" ) type localProfileDataSelection struct { @@ -31,29 +30,17 @@ func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, prof bookmarks, bookmarkCount, bookmarkErr := localbrowser.ExportBookmarks(profile) historyCount, historyErr := localbrowser.HistoryCount(ctx, profile, since) storageSites, storageErr := localbrowser.LocalStorageSites(ctx, profile) - extensions, extensionErr := localbrowser.DiscoverExtensions(profile) - extensionCapacity := storedExtensionCapacity{unlimited: true} - extensionCapacityKnown := c.extensionCapacity == nil - if extensionErr == nil && len(extensions) > 0 && c.extensionCapacity != nil { - var capacityErr error - extensionCapacity, capacityErr = c.extensionCapacity(ctx) - extensionCapacityKnown = capacityErr == nil - if capacityErr != nil && humanOutput { - pterm.Warning.Printf("extension capacity could not be checked; Kernel will enforce it during import: %v\n", capacityErr) - } - } if humanOutput { warnUnavailableBrowserData("bookmarks", bookmarkErr) warnUnavailableBrowserData("history", historyErr) warnUnavailableBrowserData("local storage", storageErr) - warnUnavailableBrowserData("extensions", extensionErr) } selection := localProfileDataSelection{history: includeHistory} - options := make([]string, 0, 4) - defaults := make([]string, 0, 4) - labels := make(map[string]string, 4) + options := make([]string, 0, 3) + defaults := make([]string, 0, 3) + labels := make(map[string]string, 3) if bookmarkErr == nil && bookmarkCount > 0 { labels[bookmarksChoice] = fmt.Sprintf("%s — %d", bookmarksChoice, bookmarkCount) options = append(options, labels[bookmarksChoice]) @@ -72,12 +59,6 @@ func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, prof options = append(options, labels[storageChoice]) defaults = append(defaults, labels[storageChoice]) } - if extensionErr == nil && len(extensions) > 0 { - labels[extensionsChoice] = fmt.Sprintf("%s — %d detected (plan limit applies)", extensionsChoice, len(extensions)) - options = append(options, labels[extensionsChoice]) - defaults = append(defaults, labels[extensionsChoice]) - } - chosen := defaults var err error if !nonInteractive && len(options) > 0 { @@ -97,8 +78,6 @@ func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, prof case labels[storageChoice]: selection.storage = true selection.storageBytes = storageSiteBytes(storageSites) - case labels[extensionsChoice]: - selection.data.Extensions = append(selection.data.Extensions, extensions...) } } @@ -109,12 +88,6 @@ func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, prof } selection.storageBytes = selectedStorageSiteBytes(storageSites, selection.storageSites) } - if len(selection.data.Extensions) > 0 { - selection.data.Extensions, err = c.chooseProfileExtensions(selection.data.Extensions, extensionCapacity, extensionCapacityKnown, nonInteractive) - if err != nil { - return localProfileDataSelection{}, err - } - } return selection, nil } @@ -141,9 +114,6 @@ func (c ProfilesImportLocalCmd) confirmBrowserImport(targetName string, cookies if profileData.storage { pterm.Printf(" Local storage — %s across %d origins\n", formatBinaryBytes(profileData.storageBytes), len(profileData.storageSites)) } - if count := len(profileData.data.Extensions); count > 0 { - pterm.Printf(" Browser extensions — %d\n", count) - } loginCount := 0 for _, provider := range logins.providers { loginCount += len(provider.candidates) @@ -212,48 +182,6 @@ func (c ProfilesImportLocalCmd) chooseLocalStorageSites(sites []localbrowser.Sto return result, nil } -func (c ProfilesImportLocalCmd) chooseProfileExtensions(extensions []localbrowser.Extension, capacity storedExtensionCapacity, capacityKnown, nonInteractive bool) ([]localbrowser.Extension, error) { - maximum := min(20, len(extensions)) - if capacityKnown && !capacity.unlimited { - maximum = min(maximum, capacity.remaining) - } - if nonInteractive { - if len(extensions) > maximum { - return nil, fmt.Errorf("%d extensions were selected, but only %d can be added under the profile and plan limits; run interactively to choose extensions", len(extensions), maximum) - } - return extensions, nil - } - labels := make([]string, 0, len(extensions)) - byLabel := make(map[string]localbrowser.Extension, len(extensions)) - for index, extension := range extensions { - name := extension.Name - if name == "" { - name = "Unnamed extension" - } - label := fmt.Sprintf("%d %s · %s", index+1, compactField(name, 42), extension.ID[len(extension.ID)-6:]) - labels = append(labels, label) - byLabel[label] = extension - } - defaults := labels[:maximum] - for { - prompt := fmt.Sprintf("Choose extensions to reinstall (select up to %d)", maximum) - chosen, err := c.prompter.MultiSelect("browser extensions", "your Kernel plan controls stored extension capacity", prompt, labels, defaults) - if err != nil { - return nil, err - } - if len(chosen) > maximum { - pterm.Warning.Printf("Select at most %d extension%s\n", maximum, pluralSuffix(maximum)) - defaults = chosen - continue - } - result := make([]localbrowser.Extension, 0, len(chosen)) - for _, label := range chosen { - result = append(result, byLabel[label]) - } - return result, nil - } -} - func buildSelectedProfileData(ctx context.Context, profile localbrowser.Profile, selection localProfileDataSelection, cookies []localbrowser.Cookie, since time.Time) (localbrowser.ProfileData, map[string]int, error) { data := selection.data data.Cookies = cookies @@ -280,14 +208,11 @@ func buildSelectedProfileData(ctx context.Context, profile localbrowser.Profile, data.Storage = storage counts["storage"] = len(storage) } - if len(data.Extensions) > 0 { - counts["extensions"] = len(data.Extensions) - } return data, counts, nil } func selectedProfileCategories(counts map[string]int) []string { - order := []string{"cookies", "storage", "bookmarks", "history", "extensions"} + order := []string{"cookies", "storage", "bookmarks", "history"} result := make([]string, 0, len(counts)) for _, category := range order { if count, selected := counts[category]; selected && count > 0 { diff --git a/cmd/browser_import_profile_data_test.go b/cmd/browser_import_profile_data_test.go index cf58bb60..da2a1d0d 100644 --- a/cmd/browser_import_profile_data_test.go +++ b/cmd/browser_import_profile_data_test.go @@ -1,7 +1,6 @@ package cmd import ( - "strings" "testing" localbrowser "github.com/kernel/cli/internal/browserimport" @@ -26,26 +25,15 @@ func TestLocalStorageSelectionRequiresReviewWhenOverLimit(t *testing.T) { require.ErrorContains(t, err, "run interactively to choose websites") } -func TestNonInteractiveExtensionSelectionHonorsCapacity(t *testing.T) { - extensions := []localbrowser.Extension{ - {ID: strings.Repeat("a", 32), Source: "chrome_web_store"}, - {ID: strings.Repeat("b", 32), Source: "chrome_web_store"}, - } - - _, err := (ProfilesImportLocalCmd{}).chooseProfileExtensions(extensions, storedExtensionCapacity{remaining: 1}, true, true) - require.ErrorContains(t, err, "only 1 can be added") -} - func TestSelectedProfileCategoriesUsePortableApplyOrder(t *testing.T) { categories := selectedProfileCategories(map[string]int{ - "extensions": 1, - "bookmarks": 2, - "cookies": 3, - "history": 4, - "storage": 5, + "bookmarks": 2, + "cookies": 3, + "history": 4, + "storage": 5, }) - require.Equal(t, []string{"cookies", "storage", "bookmarks", "history", "extensions"}, categories) + require.Equal(t, []string{"cookies", "storage", "bookmarks", "history"}, categories) } func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) { diff --git a/cmd/profiles_import_local.go b/cmd/profiles_import_local.go index 1b301a45..e373d0b5 100644 --- a/cmd/profiles_import_local.go +++ b/cmd/profiles_import_local.go @@ -52,7 +52,6 @@ type ProfilesImportLocalCmd struct { providers func() []passwordmanager.Provider provisioner managedAuthProvisioner managedAuthCapacity func(context.Context) (managedAuthCapacity, error) - extensionCapacity func(context.Context) (storedExtensionCapacity, error) } type pendingManagedAuth struct { @@ -306,9 +305,6 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI if count := itemCounts["storage"]; count > 0 { pterm.Success.Printf("Imported %d local storage keys from %d origins\n", count, importedStorageOriginCount(profileData.Storage)) } - if count := itemCounts["extensions"]; count > 0 { - pterm.Success.Printf("Installed %d browser extensions\n", count) - } } connectionIDs := make([]string, 0) approvedLogins := make([]passwordmanager.Record, 0) @@ -1520,9 +1516,6 @@ func runProfilesImportLocalWithInput(cmd *cobra.Command, input ProfilesImportLoc providers: passwordmanager.Detect, provisioner: kernelManagedAuthProvisioner{credentials: &credentials, connections: &connections}, managedAuthCapacity: func(ctx context.Context) (managedAuthCapacity, error) { return loadManagedAuthCapacity(ctx, &limits) }, - extensionCapacity: func(ctx context.Context) (storedExtensionCapacity, error) { - return loadStoredExtensionCapacity(ctx, &limits) - }, } input.ProjectID = project.ID if input.Version == "" { diff --git a/cmd/profiles_import_local_test.go b/cmd/profiles_import_local_test.go index 0a9bd3bc..9392f917 100644 --- a/cmd/profiles_import_local_test.go +++ b/cmd/profiles_import_local_test.go @@ -384,19 +384,6 @@ func TestDecodeManagedAuthCapacity(t *testing.T) { }) } -func TestDecodeStoredExtensionCapacity(t *testing.T) { - capacity, err := decodeStoredExtensionCapacity(`{"max_stored_extensions":5,"stored_extensions_used":3}`) - require.NoError(t, err) - assert.Equal(t, storedExtensionCapacity{remaining: 2}, capacity) - - capacity, err = decodeStoredExtensionCapacity(`{"max_stored_extensions":null,"stored_extensions_used":12}`) - require.NoError(t, err) - assert.Equal(t, storedExtensionCapacity{unlimited: true}, capacity) - - _, err = decodeStoredExtensionCapacity(`{"max_auth_connections":10}`) - require.ErrorContains(t, err, "does not expose stored extension capacity") -} - func TestChooseManagedAuthLoginsRejectsExplicitBatchAboveRemainingConnections(t *testing.T) { command := managedAuthTestCommand(func() []passwordmanager.Provider { return []passwordmanager.Provider{fakePasswordManager{candidates: []passwordmanager.Candidate{ diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index 11981ca4..4678b6b7 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -38,11 +38,10 @@ type BundleProfile struct { } type ProfileFiles struct { - Cookies string `json:"cookies,omitempty"` - Storage string `json:"storage,omitempty"` - Bookmarks string `json:"bookmarks,omitempty"` - History string `json:"history,omitempty"` - Extensions string `json:"extensions,omitempty"` + Cookies string `json:"cookies,omitempty"` + Storage string `json:"storage,omitempty"` + Bookmarks string `json:"bookmarks,omitempty"` + History string `json:"history,omitempty"` } type bundleFile struct { @@ -51,7 +50,7 @@ type bundleFile struct { } func BuildProfileBundle(ctx context.Context, profile Profile, targetName, version string, data ProfileData) ([]byte, error) { - if len(data.Cookies) == 0 && len(data.Storage) == 0 && data.Bookmarks == nil && len(data.History) == 0 && len(data.Extensions) == 0 { + if len(data.Cookies) == 0 && len(data.Storage) == 0 && data.Bookmarks == nil && len(data.History) == 0 { return nil, fmt.Errorf("no browser data was selected") } files := ProfileFiles{} @@ -97,12 +96,6 @@ func BuildProfileBundle(ctx context.Context, profile Profile, targetName, versio } payloads = append(payloads, bundleFile{path: files.History, data: encoded}) } - if len(data.Extensions) > 0 { - files.Extensions = "profiles/" + profile.ID + "/extensions.json" - if err := addJSON(files.Extensions, "extensions", data.Extensions); err != nil { - return nil, err - } - } manifest := Manifest{ Version: BundleVersion, Source: BundleSource{OS: "macos", HelperVersion: version}, diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index dc8b7330..a7839f71 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -15,10 +15,9 @@ import ( func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ - Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}, - Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, - Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{{Name: "bookmark_bar", Children: []BookmarkNode{{Title: "Kernel", URL: "https://onkernel.com"}}}}}, - Extensions: []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Source: "chrome_web_store"}}, + Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}, + Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}, + Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{{Name: "bookmark_bar", Children: []BookmarkNode{{Title: "Kernel", URL: "https://onkernel.com"}}}}}, }) require.NoError(t, err) @@ -42,7 +41,6 @@ func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { require.NotEmpty(t, manifest.Profiles[0].Files.Storage) require.NotEmpty(t, manifest.Profiles[0].Files.Bookmarks) require.Empty(t, manifest.Profiles[0].Files.History) - require.NotEmpty(t, manifest.Profiles[0].Files.Extensions) } func TestEncodeJSONLEnforcesPortableRecordLimits(t *testing.T) { diff --git a/internal/browserimport/chromium.go b/internal/browserimport/chromium.go index cda6ecb0..28d67105 100644 --- a/internal/browserimport/chromium.go +++ b/internal/browserimport/chromium.go @@ -41,7 +41,6 @@ const ( ) var profileIDCharacters = regexp.MustCompile(`[^a-zA-Z0-9._-]+`) -var chromeWebStoreExtensionID = regexp.MustCompile(`^[a-p]{32}$`) func DiscoverMacOSProfiles(home string) ([]Profile, error) { browsers := []Browser{ @@ -580,47 +579,6 @@ func localStorageMetadataSize(raw []byte) (int64, error) { return 0, fmt.Errorf("size field is missing") } -func DiscoverExtensions(profile Profile) ([]Extension, error) { - payload, err := readFileBounded(filepath.Join(profile.Path, "Secure Preferences"), maxDocumentBytes) - if errors.Is(err, os.ErrNotExist) { - payload, err = readFileBounded(filepath.Join(profile.Path, "Preferences"), maxDocumentBytes) - } - if errors.Is(err, os.ErrNotExist) { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("read browser extensions: %w", err) - } - var source struct { - Extensions struct { - Settings map[string]struct { - State *int `json:"state"` - FromWebStore bool `json:"from_webstore"` - Manifest struct { - Name string `json:"name"` - } `json:"manifest"` - } `json:"settings"` - } `json:"extensions"` - } - if err := json.Unmarshal(payload, &source); err != nil { - return nil, fmt.Errorf("decode browser extensions: %w", err) - } - result := make([]Extension, 0) - for id, setting := range source.Extensions.Settings { - if (setting.State != nil && *setting.State != 1) || !setting.FromWebStore || !chromeWebStoreExtensionID.MatchString(id) { - continue - } - result = append(result, Extension{ID: id, Name: setting.Manifest.Name, Source: "chrome_web_store"}) - } - sort.Slice(result, func(i, j int) bool { - if result[i].Name == result[j].Name { - return result[i].ID < result[j].ID - } - return result[i].Name < result[j].Name - }) - return result, nil -} - // CookieSites returns every website with importable cookies, ranked by recent // browser use. It reads cookie metadata only; values are decrypted by // ExportCookies after the user chooses what to import. diff --git a/internal/browserimport/chromium_test.go b/internal/browserimport/chromium_test.go index baa1ee84..52759d45 100644 --- a/internal/browserimport/chromium_test.go +++ b/internal/browserimport/chromium_test.go @@ -300,24 +300,6 @@ func TestLocalStorageSitesAndExportUseLivePortableRecords(t *testing.T) { require.Equal(t, "hello 世界", records[1].Value) } -func TestDiscoverExtensionsUsesPortableAllowlist(t *testing.T) { - profile := sqliteProfileFixture(t) - payload := `{ - "browser":{"theme":{"color_scheme2":2},"show_home_button":true}, - "extensions":{"settings":{ - "abcdefghijklmnopabcdefghijklmnop":{"from_webstore":true,"manifest":{"name":"Eligible"}}, - "bcdefghijklmnopabcdefghijklmnopa":{"state":0,"from_webstore":true,"manifest":{"name":"Disabled"}}, - "cdefghijklmnopabcdefghijklmnopab":{"state":1,"from_webstore":false,"manifest":{"name":"Local"}} - }} -}` - require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Preferences"), []byte(payload), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(profile.Path, "Secure Preferences"), []byte(payload), 0o600)) - - extensions, err := DiscoverExtensions(profile) - require.NoError(t, err) - require.Equal(t, []Extension{{ID: "abcdefghijklmnopabcdefghijklmnop", Name: "Eligible", Source: "chrome_web_store"}}, extensions) -} - func TestSelectedCookieFilterEscapesInput(t *testing.T) { cte, clause := selectedCookieFilter([]string{"example.com' OR 1=1 --"}, false) assert.Contains(t, cte, "example.com'' or 1=1 --") diff --git a/internal/browserimport/types.go b/internal/browserimport/types.go index cbce9af2..3ccdf55e 100644 --- a/internal/browserimport/types.go +++ b/internal/browserimport/types.go @@ -87,18 +87,11 @@ type StorageSite struct { Bytes int64 } -type Extension struct { - ID string `json:"id"` - Name string `json:"name,omitempty"` - Source string `json:"source"` -} - type ProfileData struct { - Cookies []Cookie - Storage []StorageRecord - Bookmarks *BookmarkDocument - History []HistoryRecord - Extensions []Extension + Cookies []Cookie + Storage []StorageRecord + Bookmarks *BookmarkDocument + History []HistoryRecord } type Source struct { @@ -139,7 +132,6 @@ type ApplyFailure struct { type Applied struct { Profiles []AppliedProfile `json:"profiles"` CredentialsImported int `json:"credentials_imported"` - ExtensionsDetected int `json:"extensions_detected"` Failure *ApplyFailure `json:"failure,omitempty"` } From 988debada7abbdd8468a8517baec0eeffdbe4699 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:05:34 -0400 Subject: [PATCH 5/8] Align browser import CLI upload limit --- internal/browserimport/bundle.go | 4 ++-- internal/browserimport/bundle_test.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index 4678b6b7..e588571f 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -11,7 +11,7 @@ import ( ) const ( - maxBundleBytes = 128 << 20 + maxBundleBytes = 64 << 20 maxPortableFileBytes = 64 << 20 maxPortableRecordBytes = 1 << 20 maxPortableRecords = 100_000 @@ -165,7 +165,7 @@ func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]b return nil, err } if output.Len() > maxBundleBytes { - return nil, fmt.Errorf("selected browser data exceeds the 128 MiB import limit") + return nil, fmt.Errorf("selected browser data exceeds the %d MiB import limit", maxBundleBytes>>20) } return output.Bytes(), nil } diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index a7839f71..defb9641 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -12,6 +12,10 @@ import ( "github.com/stretchr/testify/require" ) +func TestBundleLimitMatchesUploadContract(t *testing.T) { + require.Equal(t, 64<<20, maxBundleBytes) +} + func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ From 671e992466cdda836eca979ed6c6a509676b2faf Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:25:52 -0400 Subject: [PATCH 6/8] Fit oversized browser imports safely --- cmd/browser_import_profile_data.go | 158 ++++++++++++++++++++++++ cmd/browser_import_profile_data_test.go | 89 +++++++++++++ cmd/profiles_import_local.go | 22 +++- internal/browserimport/bundle.go | 26 +++- internal/browserimport/bundle_test.go | 12 ++ 5 files changed, 303 insertions(+), 4 deletions(-) diff --git a/cmd/browser_import_profile_data.go b/cmd/browser_import_profile_data.go index 20344a59..6dd22991 100644 --- a/cmd/browser_import_profile_data.go +++ b/cmd/browser_import_profile_data.go @@ -2,6 +2,8 @@ package cmd import ( "context" + "encoding/json" + "errors" "fmt" "sort" "time" @@ -26,6 +28,143 @@ type localProfileDataSelection struct { storageBytes int64 } +type profileBundleBuilder func(localbrowser.ProfileData) ([]byte, error) + +type profileBundleFit struct { + bundle []byte + data localbrowser.ProfileData + itemCounts map[string]int + originalSize int64 + limit int64 + skippedStorageOrigins []string + skippedStorageRecords int + skippedStorageBytes int64 + skippedHistoryRecords int +} + +type storageOriginGroup struct { + origin string + bytes int64 + records int +} + +func fitBrowserImportBundle(data localbrowser.ProfileData, itemCounts map[string]int, build profileBundleBuilder) (profileBundleFit, error) { + bundle, err := build(data) + if err == nil { + return profileBundleFit{bundle: bundle, data: data, itemCounts: cloneItemCounts(itemCounts)}, nil + } + var originalTooLarge *localbrowser.BundleTooLargeError + if !errors.As(err, &originalTooLarge) { + return profileBundleFit{}, err + } + + groups := groupedStorageOrigins(data.Storage) + fit, found, err := fitBrowserImportStorage(data, itemCounts, groups, false, originalTooLarge, build) + if err != nil { + return profileBundleFit{}, err + } + if !found && len(data.History) > 0 { + fit, found, err = fitBrowserImportStorage(data, itemCounts, groups, true, nil, build) + if err != nil { + return profileBundleFit{}, err + } + } + if !found { + return profileBundleFit{}, fmt.Errorf("cookies and bookmarks do not fit in the browser import bundle: %w", originalTooLarge) + } + fit.originalSize = originalTooLarge.Size + fit.limit = originalTooLarge.Limit + return fit, nil +} + +func fitBrowserImportStorage(data localbrowser.ProfileData, itemCounts map[string]int, groups []storageOriginGroup, dropHistory bool, pendingTooLarge *localbrowser.BundleTooLargeError, build profileBundleBuilder) (profileBundleFit, bool, error) { + fit := profileBundleFit{data: data, itemCounts: cloneItemCounts(itemCounts)} + if dropHistory { + fit.skippedHistoryRecords = len(data.History) + fit.data.History = nil + delete(fit.itemCounts, "history") + } + removed := make(map[string]struct{}, len(groups)) + nextGroup := 0 + for { + if pendingTooLarge == nil { + bundle, err := build(fit.data) + if err == nil { + fit.bundle = bundle + return fit, true, nil + } + if !errors.As(err, &pendingTooLarge) { + return profileBundleFit{}, false, err + } + } + if nextGroup == len(groups) { + return profileBundleFit{}, false, nil + } + + bytesToRemove := pendingTooLarge.Size - pendingTooLarge.Limit + var removedBytes int64 + for nextGroup < len(groups) && removedBytes < bytesToRemove { + group := groups[nextGroup] + nextGroup++ + removed[group.origin] = struct{}{} + removedBytes += group.bytes + fit.skippedStorageOrigins = append(fit.skippedStorageOrigins, group.origin) + fit.skippedStorageRecords += group.records + fit.skippedStorageBytes += group.bytes + } + fit.data.Storage = filterStorageOrigins(data.Storage, removed) + if len(fit.data.Storage) == 0 { + delete(fit.itemCounts, "storage") + } else { + fit.itemCounts["storage"] = len(fit.data.Storage) + } + pendingTooLarge = nil + } +} + +func groupedStorageOrigins(records []localbrowser.StorageRecord) []storageOriginGroup { + byOrigin := make(map[string]*storageOriginGroup) + for _, record := range records { + group := byOrigin[record.Origin] + if group == nil { + group = &storageOriginGroup{origin: record.Origin} + byOrigin[record.Origin] = group + } + encoded, _ := json.Marshal(record) + group.bytes += int64(len(encoded) + 1) + group.records++ + } + groups := make([]storageOriginGroup, 0, len(byOrigin)) + for _, group := range byOrigin { + groups = append(groups, *group) + } + sort.Slice(groups, func(left, right int) bool { + if groups[left].bytes == groups[right].bytes { + return groups[left].origin < groups[right].origin + } + return groups[left].bytes > groups[right].bytes + }) + return groups +} + +func filterStorageOrigins(records []localbrowser.StorageRecord, removed map[string]struct{}) []localbrowser.StorageRecord { + filtered := make([]localbrowser.StorageRecord, 0, len(records)) + for _, record := range records { + if _, skip := removed[record.Origin]; !skip { + filtered = append(filtered, record) + } + } + return filtered +} + +func cloneItemCounts(counts map[string]int) map[string]int { + cloned := make(map[string]int, len(counts)) + for category, count := range counts { + cloned[category] = count + } + return cloned +} + func (c ProfilesImportLocalCmd) chooseLocalProfileData(ctx context.Context, profile localbrowser.Profile, since time.Time, includeHistory, nonInteractive, humanOutput bool) (localProfileDataSelection, error) { bookmarks, bookmarkCount, bookmarkErr := localbrowser.ExportBookmarks(profile) historyCount, historyErr := localbrowser.HistoryCount(ctx, profile, since) @@ -125,6 +264,25 @@ func (c ProfilesImportLocalCmd) confirmBrowserImport(targetName string, cookies return c.prompter.ConfirmDefault("import browser data", "Proceed?", true) } +func (c ProfilesImportLocalCmd) confirmBundleFallback(fit profileBundleFit) (bool, error) { + pterm.Println() + pterm.Warning.Printf("Browser data is %s; Kernel supports %s.\n\n", formatBinaryBytes(fit.originalSize), formatBinaryBytes(fit.limit)) + pterm.Println("To fit, Kernel will skip:") + if fit.skippedStorageRecords > 0 { + originCount := len(fit.skippedStorageOrigins) + pterm.Printf( + " Local storage — %d key%s from %d origin%s (%s)\n", + fit.skippedStorageRecords, pluralSuffix(fit.skippedStorageRecords), + originCount, pluralSuffix(originCount), formatBinaryBytes(fit.skippedStorageBytes), + ) + } + if fit.skippedHistoryRecords > 0 { + pterm.Printf(" History — %d visit%s\n", fit.skippedHistoryRecords, pluralSuffix(fit.skippedHistoryRecords)) + } + pterm.Println("\nCookies and selected bookmarks will remain included.") + return c.prompter.ConfirmDefault("fit browser import", "Continue with these changes?", true) +} + func selectedCookieCount(sites []localbrowser.Site, selected []string) int { set := make(map[string]struct{}, len(selected)) for _, site := range selected { diff --git a/cmd/browser_import_profile_data_test.go b/cmd/browser_import_profile_data_test.go index da2a1d0d..a7daf765 100644 --- a/cmd/browser_import_profile_data_test.go +++ b/cmd/browser_import_profile_data_test.go @@ -1,6 +1,8 @@ package cmd import ( + "errors" + "fmt" "testing" localbrowser "github.com/kernel/cli/internal/browserimport" @@ -41,3 +43,90 @@ func TestProfilesImportLocalDefaultsHistoryOn(t *testing.T) { require.NotNil(t, flag) require.Equal(t, "true", flag.DefValue) } + +func TestFitBrowserImportBundleRemovesLargestStorageOriginsFirst(t *testing.T) { + data := localbrowser.ProfileData{ + Storage: []localbrowser.StorageRecord{ + {Origin: "https://large.example", Key: "one", Value: "a much larger local storage value"}, + {Origin: "https://small.example", Key: "two", Value: "x"}, + }, + History: []localbrowser.HistoryRecord{{URL: "https://example.com"}}, + } + counts := map[string]int{"cookies": 2, "storage": 2, "history": 1} + builder := sizedBundleBuilder(50, 5, map[string]int64{ + "https://large.example": 10, + "https://small.example": 2, + }) + + result, err := fitBrowserImportBundle(data, counts, builder) + require.NoError(t, err) + require.Equal(t, []string{"https://large.example"}, result.skippedStorageOrigins) + require.Equal(t, 1, result.skippedStorageRecords) + require.Zero(t, result.skippedHistoryRecords) + require.Len(t, result.data.Storage, 1) + require.Equal(t, "https://small.example", result.data.Storage[0].Origin) + require.Equal(t, 1, result.itemCounts["storage"]) + require.Equal(t, 1, result.itemCounts["history"]) +} + +func TestFitBrowserImportBundleDropsHistoryThenRestoresStorageThatFits(t *testing.T) { + data := localbrowser.ProfileData{ + Storage: []localbrowser.StorageRecord{ + {Origin: "https://large.example", Key: "one", Value: "a much larger local storage value"}, + {Origin: "https://small.example", Key: "two", Value: "x"}, + }, + History: []localbrowser.HistoryRecord{{URL: "https://example.com"}, {URL: "https://other.example"}}, + } + counts := map[string]int{"cookies": 2, "storage": 2, "history": 2} + builder := sizedBundleBuilder(60, 10, map[string]int64{ + "https://large.example": 3, + "https://small.example": 2, + }) + + result, err := fitBrowserImportBundle(data, counts, builder) + require.NoError(t, err) + require.Equal(t, []string{"https://large.example"}, result.skippedStorageOrigins) + require.Equal(t, 1, result.skippedStorageRecords) + require.Equal(t, 2, result.skippedHistoryRecords) + require.Len(t, result.data.Storage, 1) + require.Empty(t, result.data.History) + require.NotContains(t, result.itemCounts, "history") +} + +func TestFitBrowserImportBundleLeavesBundleAloneWhenItFits(t *testing.T) { + data := localbrowser.ProfileData{Storage: []localbrowser.StorageRecord{{Origin: "https://example.com", Key: "one", Value: "value"}}} + counts := map[string]int{"cookies": 2, "storage": 1} + + result, err := fitBrowserImportBundle(data, counts, sizedBundleBuilder(60, 0, map[string]int64{"https://example.com": 3})) + require.NoError(t, err) + require.Zero(t, result.originalSize) + require.Equal(t, data.Storage, result.data.Storage) + require.Equal(t, counts, result.itemCounts) +} + +func TestFitBrowserImportBundleRejectsOversizedRequiredData(t *testing.T) { + _, err := fitBrowserImportBundle(localbrowser.ProfileData{}, map[string]int{"cookies": 2}, sizedBundleBuilder(65, 0, nil)) + require.ErrorContains(t, err, "cookies and bookmarks do not fit") + require.True(t, errors.Is(err, localbrowser.ErrBundleTooLarge)) +} + +func sizedBundleBuilder(base, history int64, storage map[string]int64) profileBundleBuilder { + return func(data localbrowser.ProfileData) ([]byte, error) { + size := base + if len(data.History) > 0 { + size += history + } + seen := make(map[string]struct{}) + for _, record := range data.Storage { + if _, ok := seen[record.Origin]; ok { + continue + } + seen[record.Origin] = struct{}{} + size += storage[record.Origin] + } + if size > 64 { + return nil, &localbrowser.BundleTooLargeError{Size: size, Limit: 64} + } + return []byte(fmt.Sprintf("%d", size)), nil + } +} diff --git a/cmd/profiles_import_local.go b/cmd/profiles_import_local.go index e373d0b5..92b080e3 100644 --- a/cmd/profiles_import_local.go +++ b/cmd/profiles_import_local.go @@ -244,12 +244,30 @@ func (c ProfilesImportLocalCmd) Run(ctx context.Context, in ProfilesImportLocalI if err != nil { return err } - categories := selectedProfileCategories(itemCounts) - bundle, err := localbrowser.BuildProfileBundle(ctx, profile, targetName, version, profileData) + fit, err := fitBrowserImportBundle(profileData, itemCounts, func(candidate localbrowser.ProfileData) ([]byte, error) { + return localbrowser.BuildProfileBundle(ctx, profile, targetName, version, candidate) + }) timings["bundle"] = time.Since(phaseStarted) if err != nil { return err } + if fit.originalSize > 0 { + if nonInteractive { + return fmt.Errorf("%w; run interactively to review optional browser data that can be skipped", &localbrowser.BundleTooLargeError{Size: fit.originalSize, Limit: fit.limit}) + } + proceed, err := c.confirmBundleFallback(fit) + if err != nil { + return err + } + if !proceed { + pterm.Info.Println("Browser import canceled; no Kernel resources were changed") + return nil + } + } + profileData = fit.data + itemCounts = fit.itemCounts + bundle := fit.bundle + categories := selectedProfileCategories(itemCounts) token, err := auth.BearerToken(ctx) if err != nil { return err diff --git a/internal/browserimport/bundle.go b/internal/browserimport/bundle.go index e588571f..738733e1 100644 --- a/internal/browserimport/bundle.go +++ b/internal/browserimport/bundle.go @@ -5,11 +5,29 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "github.com/klauspost/compress/zstd" ) +// ErrBundleTooLarge identifies a complete compressed bundle that exceeds the upload limit. +var ErrBundleTooLarge = errors.New("browser import bundle is too large") + +// BundleTooLargeError reports the compressed bundle size and allowed limit. +type BundleTooLargeError struct { + Size int64 + Limit int64 +} + +func (e *BundleTooLargeError) Error() string { + return fmt.Sprintf("selected browser data is %.1f MiB; Kernel supports %.0f MiB: %v", float64(e.Size)/(1<<20), float64(e.Limit)/(1<<20), ErrBundleTooLarge) +} + +func (e *BundleTooLargeError) Unwrap() error { + return ErrBundleTooLarge +} + const ( maxBundleBytes = 64 << 20 maxPortableFileBytes = 64 << 20 @@ -134,6 +152,10 @@ func encodeJSONL[T any](label string, records []T) ([]byte, error) { } func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]byte, error) { + return encodeBundleWithLimit(ctx, manifest, files, maxBundleBytes) +} + +func encodeBundleWithLimit(ctx context.Context, manifest []byte, files []bundleFile, maxBytes int64) ([]byte, error) { var output bytes.Buffer zstdWriter, err := zstd.NewWriter(&output, zstd.WithEncoderConcurrency(1)) if err != nil { @@ -164,8 +186,8 @@ func encodeBundle(ctx context.Context, manifest []byte, files []bundleFile) ([]b if err := zstdWriter.Close(); err != nil { return nil, err } - if output.Len() > maxBundleBytes { - return nil, fmt.Errorf("selected browser data exceeds the %d MiB import limit", maxBundleBytes>>20) + if int64(output.Len()) > maxBytes { + return nil, &BundleTooLargeError{Size: int64(output.Len()), Limit: maxBytes} } return output.Bytes(), nil } diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index defb9641..9f170cab 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "encoding/json" + "errors" "io" "strings" "testing" @@ -16,6 +17,17 @@ func TestBundleLimitMatchesUploadContract(t *testing.T) { require.Equal(t, 64<<20, maxBundleBytes) } +func TestEncodeBundleReportsActualCompressedSize(t *testing.T) { + _, err := encodeBundleWithLimit(t.Context(), []byte(`{"version":1}`), nil, 1) + require.Error(t, err) + + var tooLarge *BundleTooLargeError + require.ErrorAs(t, err, &tooLarge) + require.Equal(t, int64(1), tooLarge.Limit) + require.Greater(t, tooLarge.Size, tooLarge.Limit) + require.True(t, errors.Is(err, ErrBundleTooLarge)) +} + func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ From fc628b5e9b08188880bacdbfc3e1c8c663608917 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:52:56 -0400 Subject: [PATCH 7/8] Test browser import size message --- internal/browserimport/bundle_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index 9f170cab..7a43ecb8 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -28,6 +28,12 @@ func TestEncodeBundleReportsActualCompressedSize(t *testing.T) { require.True(t, errors.Is(err, ErrBundleTooLarge)) } +func TestBundleTooLargeErrorReportsMiBWithoutLosingCause(t *testing.T) { + err := &BundleTooLargeError{Size: 96 << 20, Limit: 64 << 20} + require.Equal(t, "selected browser data is 96.0 MiB; Kernel supports 64 MiB: browser import bundle is too large", err.Error()) + require.ErrorIs(t, err, ErrBundleTooLarge) +} + func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{ From e3e3948f09503a185ff79c296ac9474a4faa2131 Mon Sep 17 00:00:00 2001 From: Ilyaas Kapadia <86218345+IlyaasK@users.noreply.github.com> Date: Thu, 20 Aug 2026 10:59:40 -0400 Subject: [PATCH 8/8] Test single-category browser imports --- internal/browserimport/bundle_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/browserimport/bundle_test.go b/internal/browserimport/bundle_test.go index 7a43ecb8..b2b2608b 100644 --- a/internal/browserimport/bundle_test.go +++ b/internal/browserimport/bundle_test.go @@ -34,6 +34,23 @@ func TestBundleTooLargeErrorReportsMiBWithoutLosingCause(t *testing.T) { require.ErrorIs(t, err, ErrBundleTooLarge) } +func TestBuildProfileBundleAcceptsEachCategoryAlone(t *testing.T) { + profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} + tests := map[string]ProfileData{ + "cookies": {Cookies: []Cookie{{Domain: ".example.com", Path: "/", Name: "session", Value: "secret"}}}, + "storage": {Storage: []StorageRecord{{Origin: "https://example.com", Kind: StorageKindLocal, Key: "theme", Value: "dark"}}}, + "bookmarks": {Bookmarks: &BookmarkDocument{Roots: []BookmarkRoot{}}}, + "history": {History: []HistoryRecord{{URL: "https://example.com", Title: "Example"}}}, + } + for name, data := range tests { + t.Run(name, func(t *testing.T) { + bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", data) + require.NoError(t, err) + require.NotEmpty(t, bundle) + }) + } +} + func TestBuildProfileBundleIncludesOnlySelectedCategories(t *testing.T) { profile := Profile{ID: "helium-default-1234", Name: "Personal", Browser: Browser{ID: "helium", Name: "Helium"}} bundle, err := BuildProfileBundle(t.Context(), profile, "my-browser", "test", ProfileData{