Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 40x 40x 40x 6x 6x 34x 40x 5x 5x 29x 29x 29x 1x 1x 1x 1x 1x 1x 1x 1x 35x 35x 12x 12x 23x 23x 23x 1x 1x 1x 1x 1x 1x 1x 1x 5x 5x 5x 5x 5x 5x 1x 1x 5x 1x 1x 1x 1x 1x 1x 1x 1x 7x 7x 7x 7x 2x 2x 5x 5x 7x 1x 1x 4x 4x 4x 4x 4x 4x 1x 1x 1x 1x 1x 1x 1x 1x 5x 5x 5x 5x 5x 1x 1x 5x 1x 1x 1x 1x 1x 1x 1x 1x 11x 11x 11x 11x 11x 1x 1x 10x 10x 11x 1x 1x 9x 9x 9x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 212x 212x 212x 212x 2968x 14x 14x 14x 14x 6x 6x 6x 8x 14x 7x 7x 7x 1x 1x 1x 1x 1x 2968x 212x 212x 1x 1x 1x 1x 1x 1x 184x 184x 184x 184x 184x 2x 2x 184x 184x 3x 3x 184x 184x 1x 1x 184x 184x 1x 1x 184x 184x 184x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 212x 212x 28x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 212x 55x 55x 55x 212x 212x 212x 212x 212x 47x 47x 47x 212x 212x 212x 212x 212x 48x 48x 48x 212x 212x 212x 212x 212x 34x 34x 34x 212x 212x 212x 212x 212x 212x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 191x 191x 191x 181x 181x 10x 10x 10x 10x 10x 191x 8x 8x 8x 191x 1x 1x 1x 1x 1x 1x 1x 212x 212x 212x 184x 184x 184x 184x 5x 5x 184x 212x 212x 28x 28x 212x 212x | import {input, password} from "@inquirer/prompts";
import {Command, CommanderError, InvalidArgumentError, Option} from "commander";
import {Cron} from "croner";
import {readFileSync} from "fs";
import {Resources} from "../lib/resources/main.js";
import {APP_ERR} from "./error/error-codes.js";
import {iCPSError} from "./error/error.js";
import {ArchiveApp, DaemonApp, iCPSApp, SyncApp, TokenApp} from "./icloud-app.js";
import {LogLevel} from "../lib/resources/state-manager.js";
import {TRASH_DIR} from "../lib/photos-library/constants.js";
/**
* This function can be used as a commander argParser. It will try to parse the value as a positive integer and throw an invalid argument error in case it fails
* @param value - The string literal, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns The parsed number
* @throws An InvalidArgumentError in case parsing failed
*/
function commanderParsePositiveInt(value: string, _dummyPrevious?: unknown): number {
const parsedValue = parseInt(value, 10);
if (isNaN(parsedValue)) {
throw new InvalidArgumentError(`Not a number.`);
}
if (parsedValue < 0) {
throw new InvalidArgumentError(`Not a positive number.`);
}
return parsedValue;
}
/**
* This function can be used as a commander argParser. It will try to parse the value as an integer or the string literal \`Infinity\` and throw an invalid argument error in case it fails
* @param value - The string literal, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns The parsed number
* @throws An InvalidArgumentError in case parsing failed
*/
function commanderParsePositiveIntOrInfinity(value: string, _dummyPrevious?: unknown): number {
if (value === `Infinity`) {
return Infinity;
}
return commanderParsePositiveInt(value);
}
/**
* Tries parsing a string as Cron schedule
* @param value - The string literal, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns The original string
* @throws An InvalidArgumentError in case parsing failed
*/
function commanderParseCron(value: string, _dummyPrevious?: unknown): string {
try {
const job = new Cron(value);
job.stop();
return value;
} catch (_err) {
throw new InvalidArgumentError(`Not a valid cron pattern. See https://crontab.guru (or for more information on the underlying implementation https://github.com/hexagon/croner#pattern).`);
}
}
/**
* This function can be used as a commander argParser. It will try to parse the value as an interval with the format \<numberOfRequests|Infinity\>/\<timeInMs\>.
* @param value - The string literal, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns A tuple consisting of the max number of runs in the given interval of time and the interval in ms.
* @throws An InvalidArgumentError in case parsing failed
*/
function commanderParseInterval(value: string, _dummyPrevious?: unknown): [number, number] {
const expectedRegExp = /^([0-9]+|Infinity)\/([0-9]+)$/;
const match = value.match(expectedRegExp);
if (!match) {
throw new InvalidArgumentError(`Not a valid interval pattern. Expects the format '<numberOfRequests|Infinity>/<timeInMs>', e.g. '1/20' to limit requests to one in 20ms.`);
}
const intervalCap = commanderParsePositiveIntOrInfinity(match[1]);
if (typeof intervalCap === `number` && intervalCap <= 0) {
throw new InvalidArgumentError(`Not a valid interval. Number of runs needs to be >0.`);
}
return [
intervalCap,
commanderParsePositiveInt(match[2]),
];
}
/**
* This function can be used as a commander argParser. It will try to parse the value as an URL and throw an invalid argument error in case it fails
* @param value - The string literal, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns the original value
* @throws An InvalidArgumentError in case parsing failed
*/
function commanderParseUrl(value: string, _dummyPrevious?: unknown): string {
try {
const _url = new URL(value);
return value;
} catch (err) {
throw new InvalidArgumentError(`Not a valid URL: ${err}`);
}
}
/**
* This function can be used as a commander argParser. It will read the content of the file at the provided path (e.g. a Docker secret) and throw an invalid argument error in case it fails. Trailing line breaks are removed, any other whitespace is preserved.
* @param value - The file path, read from the CLI
* @param _dummyPrevious - Conforming to the interface - unused
* @returns The content of the file
* @throws An InvalidArgumentError in case the file cannot be read or is empty
*/
function commanderReadFile(value: string, _dummyPrevious?: unknown): string {
let content: string;
try {
content = readFileSync(value, {encoding: `utf-8`});
} catch (err) {
throw new InvalidArgumentError(`Unable to read file: ${(err as Error).message}`);
}
content = content.replace(/(\r?\n)+$/, ``);
if (content.length === 0) {
throw new InvalidArgumentError(`File is empty.`);
}
return content;
}
/**
* Accepted (lower case) values of environment variables for boolean options
*/
const BOOLEAN_ENV_VALUES = {
true: [`true`, `1`, `yes`, `on`],
false: [`false`, `0`, `no`, `off`, ``],
};
/**
* Commander enables a boolean option, as soon as its environment variable is defined - regardless of its value.
* This function registers listeners on the provided command, in order to parse the value of those environment variables instead (e.g. `false` disables the option).
* The listeners are invoked after commander's own listener and overwrite its value.
* @param program - The commander command, after all options were added
*/
function commanderParseBooleanEnv(program: Command) {
program.options
.filter(option => option.isBoolean() && option.envVar !== undefined)
.forEach(option => {
program.on(`optionEnv:${option.name()}`, () => {
const envValue = process.env[option.envVar!] ?? ``;
const normalizedValue = envValue.trim().toLowerCase();
if (BOOLEAN_ENV_VALUES.true.includes(normalizedValue)) {
program.setOptionValueWithSource(option.attributeName(), true, `env`);
return;
}
if (BOOLEAN_ENV_VALUES.false.includes(normalizedValue)) {
program.setOptionValueWithSource(option.attributeName(), false, `env`);
return;
}
program.error(
`error: environment variable '${option.envVar}' value '${envValue}' is invalid. Expected one of ${[...BOOLEAN_ENV_VALUES.true, ...BOOLEAN_ENV_VALUES.false.filter(value => value.length > 0)].join(`, `)} (or an empty value for false).`,
{code: `commander.invalidArgument`},
);
});
});
}
/**
* Extracts the options from the parsed commander command - and asks for user input in case it is necessary
* @param parsedCommand - The parsed commander command returned from callback in Command.action((_, command any)
* @returns Validated iCPSAppOptions
*/
async function completeConfigurationOptionsFromCommand(parsedCommand: unknown): Promise<iCPSAppOptions> {
const {usernameFile, passwordFile, ...opts} = (parsedCommand as any).parent?.opts() as iCPSAppOptions & {usernameFile?: string, passwordFile?: string};
// Commander makes sure that only one of the options was provided - file content was read during parsing
if (usernameFile !== undefined) {
opts.username = usernameFile;
}
if (passwordFile !== undefined) {
opts.password = passwordFile;
}
while (!opts.username || opts.username.length === 0) {
opts.username = await input({message: `Please enter your AppleID username`});
}
while (!opts.password || opts.password.length === 0) {
opts.password = await password({message: `Please enter your AppleID password`, mask: `*`, toggleMask: true});
}
return opts;
}
/**
* Typed available app options
*/
export type iCPSAppOptions = {
username: string,
password: string,
trustToken?: string,
dataDir: string,
port: number,
webBasePath: string,
maxRetries: number,
downloadThreads: number,
downloadTimeout: number,
schedule: string,
enableCrashReporting: boolean,
enableNetworkCapture: boolean,
useSystemProxy: boolean,
mfaTimeout: number,
force: boolean,
refreshToken: boolean,
remoteDelete: boolean,
syncHidden: boolean,
softDelete: boolean,
trashDir: string,
logLevel: LogLevel,
silent: boolean,
logToCli: boolean,
suppressWarnings: boolean,
exportMetrics: boolean,
exportPrometheusMetrics: boolean,
region: Resources.Types.Region,
legacyLogin: boolean,
metadataRate: [number, number],
healthCheckUrl?: string,
}
/**
* Creates the argument parser for the CLI and environment variables
* @param callback - A callback function that will be called with the created app, based on the provided options.
* @returns The commander command object, awaiting .parse() to be called
*/
export function argParser(callback: (res: iCPSApp) => void): Command {
// Overwriting commander\`s _exit function, because exitOverwrite will still call process.exit
(Command.prototype as any)._exit = (exitCode: number, code: string, message: string) => {
throw new CommanderError(exitCode, code, message); // Function needs to return \`never\`, otherwise errors will be ignored
};
const program = new Command();
program.name(Resources.PackageInfo.name)
.description(Resources.PackageInfo.description)
.version(Resources.PackageInfo.version)
.addHelpText(`after`, `\nFind the full documentation at https://icps.steiler.dev/`)
.addOption(new Option(`-u, --username <string>`, `AppleID username. Omitting the option will result in the CLI to ask for user input before startup.`)
.env(`APPLE_ID_USER`)
.makeOptionMandatory(false))
.addOption(new Option(`-p, --password <string>`, `AppleID password. Omitting the option will result in the CLI to ask for user input before startup.`)
.env(`APPLE_ID_PWD`)
.makeOptionMandatory(false))
.addOption(new Option(`--username-file <path>`, `Path to a file containing the AppleID username (e.g. a Docker secret), as an alternative to the username option. Trailing line breaks are removed.`)
.env(`APPLE_ID_USER_FILE`)
.conflicts(`username`)
.argParser(commanderReadFile))
.addOption(new Option(`--password-file <path>`, `Path to a file containing the AppleID password (e.g. a Docker secret), as an alternative to the password option. Trailing line breaks are removed.`)
.env(`APPLE_ID_PWD_FILE`)
.conflicts(`password`)
.argParser(commanderReadFile))
.addOption(new Option(`-T, --trust-token <string>`, `The trust token for authentication. If not provided, the trust token is read from the \`.icloud-photos-sync\` resource file in data dir. If no stored trust token could be loaded, a new trust token will be acquired (requiring the input of an MFA code).`)
.env(`TRUST_TOKEN`))
.addOption(new Option(`-d, --data-dir <string>`, `Directory to store local copy of library.`)
.env(`DATA_DIR`)
.default(`/opt/icloud-photos-library`))
.addOption(new Option(`-P, --port <number>`, `Port to serve the web ui on and to receive MFA input on.`)
.env(`PORT`)
.default(80)
.argParser(commanderParsePositiveInt))
.addOption(new Option(`--web-base-path <string>`, `The base path for subpath deployments, e.g. '/somePath' if the application is hosted under 'http://localhost/somepath/'.`)
.env(`WEB_BASE_PATH`)
.default(``))
.addOption(new Option(`-r, --max-retries <number>`, `Sets the number of maximum retries upon a sync error (\`Infinity\` means that it will always retry).`)
.env(`MAX_RETRIES`)
.default(10)
.argParser(commanderParsePositiveIntOrInfinity))
.addOption(new Option(`-t, --download-threads <number>`, `Sets the number of concurrent download threads (\`Infinity\` will remove all limitations).`)
.env(`DOWNLOAD_THREADS`)
.default(5)
.argParser(commanderParsePositiveIntOrInfinity))
.addOption(new Option(`--download-timeout <number>`, `Sets the timeout (in minutes) for downloading assets, because downloads sometimes hang. Should be increased on slower connections and/or if there are large assets in the library (\`Infinity\` will remove the timeout).`)
.env(`DOWNLOAD_TIMEOUT`)
.default(10)
.argParser(commanderParsePositiveIntOrInfinity))
.addOption(new Option(`-S, --schedule <cron-string>`, `In case this app is executed in daemon mode, it will use this cron schedule to perform regular sync operations.`)
.env(`SCHEDULE`)
.default(`0 2 * * *`)
.argParser(commanderParseCron))
.addOption(new Option(`--enable-crash-reporting`, `Enables automatic collection of errors and crashes, see https://icps.steiler.dev/user-guides/error-reporting/ for more information.`)
.env(`ENABLE_CRASH_REPORTING`)
.default(false))
.addOption(new Option(`--mfa-timeout <number>`, `If a MFA code is necessary to authenticate, wait for these many seconds before canceling the authentication process. Time in seconds, should not exceed 10mins (due to server side timing).`)
.env(`MFA_TIMEOUT`)
.default(60 * 10)
.argParser(commanderParsePositiveInt))
.addOption(new Option(`--force`, `Forcefully remove an existing library lock. USE WITH CAUTION!`)
.env(`FORCE`)
.default(false))
.addOption(new Option(`--refresh-token`, `Invalidate any stored trust token upon startup.`)
.env(`REFRESH_TOKEN`)
.default(false))
.addOption(new Option(`--remote-delete`, `If this flag is set, delete non-favorite photos in the iCloud Photos backend upon archiving.`)
.env(`REMOTE_DELETE`)
.default(false))
.addOption(new Option(`--sync-hidden`, `If this flag is set, photos from the 'Hidden' album are synced as well. They are linked into the albums they belong to and into the \`_Hidden-Photos\` album.`)
.env(`SYNC_HIDDEN`)
.default(false))
.addOption(new Option(`--soft-delete`, `If this flag is set, assets that were deleted in iCloud Photos are moved to the trash directory (see \`--trash-dir\`), instead of being permanently deleted from disk.`)
.env(`SOFT_DELETE`)
.default(false))
.addOption(new Option(`--trash-dir <path>`, `Directory to move deleted assets to, if soft delete is enabled. Relative paths are resolved against the data dir. The trash is never cleaned automatically.`)
.env(`TRASH_DIR`)
.default(TRASH_DIR))
.addOption(new Option(`-l, --log-level <level>`, `Set the log level.`)
.env(`LOG_LEVEL`)
.choices(Object.values(LogLevel))
.default(`info`))
.addOption(new Option(`-s, --silent`, `Disables all output to the console.`)
.env(`SILENT`)
.default(false))
.addOption(new Option(`--log-to-cli`, `Disables logging to file and logs everything to the console.`)
.env(`LOG_TO_CLI`)
.default(false))
.addOption(new Option(`--suppress-warnings`, `Non critical warnings will not be displayed in the UI. They will still go into the log.`)
.env(`SUPPRESS_WARNINGS`)
.default(false))
.addOption(new Option(`--export-metrics`, `Enables the export of sync metrics to a file using the Influx Line Protocol. Written to \`.icloud-photos-sync.metrics\` in the data dir.`)
.env(`EXPORT_METRICS`)
.default(false))
.addOption(new Option(`--export-prometheus-metrics`, `Exposes sync metrics in the Prometheus/OpenMetrics format on the \`/metrics\` endpoint of the web server.`)
.env(`EXPORT_PROMETHEUS_METRICS`)
.default(false))
.addOption(new Option(`--enable-network-capture`, `Enables network capture, and generate a HAR file for debugging purposes. Written to \`.icloud-photos-sync.har\` in the data dir.`)
.env(`ENABLE_NETWORK_CAPTURE`)
.default(false))
.addOption(new Option(`--use-system-proxy`, `Routes all requests through the proxy configured by the \`HTTP_PROXY\` and \`HTTPS_PROXY\` environment variables (honouring \`NO_PROXY\`). Proxy variables are ignored otherwise.`)
.env(`USE_SYSTEM_PROXY`)
.default(false))
.addOption(new Option(`--metadata-rate <interval>`, `Limits the rate of metadata fetching in order to avoid getting throttled by the API. Expects the format \`<numberOfRequests|Infinity>/<timeInMs>\`, e.g. \`1/20\` to limit requests to one request in 20ms.`)
.env(`METADATA_RATE`)
.default([Infinity, 0], `Infinity/0`)
.argParser(commanderParseInterval))
.addOption(new Option(`--region <string>`, `Changes the iCloud region.`)
.env(`REGION`)
.default(`world`)
.choices(Object.values(Resources.Types.Region)))
.addOption(new Option(`--legacy-login`, `Enables plain text legacy login method.`)
.env(`LEGACY_LOGIN`)
.default(false))
.addOption(new Option(`--health-check-url <url>`, `URL to ping to monitor the health of icloud photos sync, see https://icps.steiler.dev/user-guides/health-checks/ for more information.`)
.env(`HEALTH_CHECK_URL`)
.default(undefined)
.argParser(commanderParseUrl));
commanderParseBooleanEnv(program);
program.command(`daemon`, {isDefault: true})
.action(async (_, command) => {
const opts = await completeConfigurationOptionsFromCommand(command);
Resources.setup(opts);
callback(new DaemonApp());
})
.description(`Starts the synchronization in scheduled daemon mode - continuously running based on the provided cron schedule.`);
program.command(`token`)
.action(async (_, command) => {
const opts = await completeConfigurationOptionsFromCommand(command);
Resources.setup(opts);
callback(new TokenApp());
})
.description(`Validates the current trust token, fetches a new one (if necessary) and prints it to the CLI.`);
program.command(`sync`)
.action(async (_, command) => {
const opts = await completeConfigurationOptionsFromCommand(command);
Resources.setup(opts);
callback(new SyncApp());
})
.description(`Fetches the remote state and persist it to the local disk once.`);
program.command(`archive`)
.action(async (archivePath, _, command) => {
const opts = await completeConfigurationOptionsFromCommand(command);
Resources.setup(opts);
callback(new ArchiveApp(archivePath));
})
.description(`Archives a given folder. Before archiving, it will first perform a sync, to make sure the correct state is archived.`)
.argument(`<path>`, `Path to the folder that should be archived`);
return program;
}
/**
* Permissions required when running with Node's permission model (`--permission`), mapped to the flag granting them.
* Node only permits `fs.symlink` (used for linking albums) with unrestricted file system read and write access.
*/
const REQUIRED_PERMISSIONS = {
'fs.read': `--allow-fs-read=*`,
'fs.write': `--allow-fs-write=*`,
net: `--allow-net`,
};
/**
* Makes sure the application is able to operate, if Node's permission model is enabled - instead of failing during the sync.
* In audit mode (`--permission-audit`), access is not denied, therefore no permissions are required.
* @throws An iCPSError, if required permissions are missing
*/
export function validatePermissions() {
const {permission} = process;
const nodeOptions = [...process.execArgv, ...(process.env.NODE_OPTIONS?.split(/\s+/) ?? [])];
if (!permission || nodeOptions.includes(`--permission-audit`)) {
return;
}
const missingFlags = Object.entries(REQUIRED_PERMISSIONS)
.filter(([scope]) => !permission.has(scope))
.map(([, flag]) => flag);
if (missingFlags.length > 0) {
throw new iCPSError(APP_ERR.INSUFFICIENT_PERMISSIONS)
.addMessage(`missing ${missingFlags.join(` `)}`);
}
}
/**
* This function will parse the provided string array and environment variables and return the correct application object.
* @param argv - The argument vector to be parsed
* @returns - A promise that resolves to the correct application object. Once the promise resolves, the global resource singleton will also be available. If the program is not able to parse the options, or required options are missing, an error message is printed to stderr and the promise rejects with a CommanderError.
*/
export async function appFactory(argv: string[]): Promise<iCPSApp> {
return new Promise((resolve, reject) => {
try {
argParser(async (res: iCPSApp) => {
try {
validatePermissions()
await Resources.state().acquireLibraryLock()
} catch (err) {
reject(err)
}
resolve(res);
}).parse(argv);
} catch (err) {
reject(err);
}
});
} |