marcoscostadev/kamiyomu repository overview

services:
  kamiyomu:
    image: marcoscostadev/kamiyomu:latest # Check releases for latest versions
    ports:
      - "8080:8080" # HTTP Port
    environment:
        # List of Hangfire server identifiers available to process jobs.
        # Each name corresponds to a distinct background worker instance;
        # add more entries here if you want multiple servers to share or divide queues.
        # add more entries using incrementing indexes (e.g., Worker__ServerAvailableNames__1, Worker__ServerAvailableNames__2, etc.)
        Worker__ServerAvailableNames__0:   "KamiYomu-background-1" 
        

        # Queues dedicated to downloading individual chapters.
        # add more entries using incrementing indexes (e.g., Worker__DownloadChapterQueues__1, Worker__DownloadChapterQueues__2, etc.)
        Worker__DownloadChapterQueues__0:  "download-chapter-queue-1" 

        # Queues dedicated to scheduling manga downloads (manages chapter download jobs).
        # add more entries using incrementing indexes (e.g., Worker__MangaDownloadSchedulerQueues__1, Worker__MangaDownloadSchedulerQueues__2, etc.)
        Worker__MangaDownloadSchedulerQueues__0:  "manga-download-scheduler-queue-1" 

        # Queues dedicated to discovering new chapters (polling or scraping for updates).
        # add more entries using incrementing indexes (e.g., Worker__DiscoveryNewChapterQueues__1, Worker__DiscoveryNewChapterQueues__2, etc.)
        Worker__DiscoveryNewChapterQueues__0:  "discovery-new-chapter-queue-1" 

        # Specifies the number of background processing threads Hangfire will spawn.
        # Increasing this value allows more jobs to run concurrently, but also raises CPU load 
        # and memory usage.
        # Each worker consumes ~80 MB of memory on average while active 
        # (actual usage may vary depending on the crawler agent implementation and system configuration).
        Worker__WorkerCount: 1

        # Defines the maximum number of crawler instances allowed to run concurrently for the same source.
        # Typically set to 1 to ensure only a single crawler operates at a time, preventing duplicate work,
        # resource conflicts, and potential rate‑limiting or blocking by the target system.
        # This value can be increased to improve throughput if the source supports multiple concurrent requests.
        #
        # Note:
        # - Worker__WorkerCount controls the total number of threads available.
        # - Worker__MaxConcurrentCrawlerInstances limits how many threads can be used by the same crawler.
        #
        # Examples:
        # - If Worker__MaxConcurrentCrawlerInstances = 1 and Worker__WorkerCount = 4,
        #   then up to 4 different crawler agents can run independently.
        # - If Worker__MaxConcurrentCrawlerInstances = 2 and Worker__WorkerCount = 6,
        #   then each crawler agent can run up to 2 instances concurrently,
        #   while up to 3 different crawler agents may be active at the same time.
        Worker__MaxConcurrentCrawlerInstances: 1

        # Minimum delay (in milliseconds) between job executions.
        # Helps throttle requests to external services and avoid hitting rate limits (e.g., HTTP 423 "Too Many Requests").
        Worker__MinWaitPeriodInMilliseconds: 3000

        # Maximum delay (in milliseconds) between job executions.
        # Provides variability in scheduling to reduce the chance of IP blocking or service throttling.
        Worker__MaxWaitPeriodInMilliseconds: 9001

        # Maximum number of retry attempts for failed jobs before marking them as permanently failed.
        Worker__MaxRetryAttempts: 10

        # Default language for the web interface (e.g., "en", "pt-BR", "fr").
        UI__DefaultLanguage: "en" 
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/healthz"]
      interval: 30s
      timeout: 10s
      retries: 3
    volumes:
      - ./AppData/manga:/manga # Your desired local path for manga storage
      - Kamiyomu_database:/db
      - kamiyomu_agents:/agents
      - kamiyomu_logs:/logs

volumes:
  kamiyomu_agents:
  Kamiyomu_database:
  kamiyomu_logs:

    docker-compose up -d

    git clone https://github.com/KamiYomu/KamiYomu.Web.git

/// <summary>
/// Defines a contract for manga crawling agents that support search, retrieval, and metadata extraction.
/// </summary>
public interface ICrawlerAgent : IDisposable
{
    /// <summary>
    /// Asynchronously retrieves the favicon URI associated with the crawler's target site.
    /// </summary>
    /// <param name="cancellationToken">Optional token to cancel the operation.</param>
    /// <returns>A <see cref="Task{Uri}"/> representing the favicon location.</returns>
    Task<Uri> GetFaviconAsync(CancellationToken cancellationToken);

    /// <summary>
    /// Searches for manga titles matching the specified name, using either traditional pagination or a continuation token.
    /// </summary>
    /// <param name="titleName">The title or keyword to search for.</param>
    /// <param name="paginationOptions">Pagination parameters, supporting both page-based and continuation token-based pagination.</param>
    /// <param name="cancellationToken">Optional token to cancel the operation.</param>
    /// <returns>A paged result containing a collection of matching <see cref="Manga"/> entries.</returns>
    Task<PagedResult<Manga>> SearchAsync(string titleName, PaginationOptions paginationOptions, CancellationToken cancellationToken);

    /// <summary>
    /// Retrieves detailed information about a specific manga by its unique identifier.
    /// </summary>
    /// <param name="id">The unique ID of the manga.</param>
    /// <param name="cancellationToken">Optional token to cancel the operation.</param>
    /// <returns>A <see cref="Task{Manga}"/> containing the manga details.</returns>
    Task<Manga> GetByIdAsync(string id, CancellationToken cancellationToken);

    /// <summary>
    /// Retrieves a paged list of chapters for the specified manga.
    /// </summary>
    /// <param name="manga">The manga object.</param>
    /// <param name="paginationOptions">Pagination parameters, supporting both page-based and continuation token-based pagination.</param>
    /// <returns>A paged result containing a collection of <see cref="Chapter"/> entries.</returns>
    Task<PagedResult<Chapter>> GetChaptersAsync(Manga manga, PaginationOptions paginationOptions, CancellationToken cancellationToken);
    /// <summary>
    /// Retrieves the list of page images associated with a given manga chapter.
    /// </summary>
    /// <param name="chapter">The chapter entity containing metadata and identifiers.</param>
    /// <param name="cancellationToken">Optional token to cancel the operation.</param>
    /// <returns>A collection of <see cref="Page"/> objects representing individual chapter pages.</returns>
    Task<IEnumerable<Page>> GetChapterPagesAsync(Chapter chapter, CancellationToken cancellationToken);
}

	<PropertyGroup>
		<Title>My Crawler Agent</Title>
		<Description>A dedicated crawler agent for accessing public data from My Personal Stuff. Built on KamiYomu.CrawlerAgents.Core, it enables efficient search, metadata extraction, and integration with the KamiYomu platform.</Description>
		<Authors>MyName</Authors>
		<Owners>MyName</Owners>
		<PackageProjectUrl>https://github.com/MyProjectUrl</PackageProjectUrl>
		<RepositoryUrl>https://github.com/MyRepositoryUrl</RepositoryUrl>
		<RepositoryType>git</RepositoryType>
		<PackageTags>kamiyomu-crawler-agents;manga-download</PackageTags>
		<PackageLicenseExpression>GPL-3.0-only</PackageLicenseExpression>
		<Copyright>© Personal. Licensed under GPL-3.0.</Copyright>
		<PackageIconUrl>https://raw.githubusercontent.com/MyPackageLogoUrl</PackageIconUrl>
		<PackageIcon>Resources/logo.png</PackageIcon>
		<PackageReadmeFile>README.md</PackageReadmeFile>
	</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)' == 'Debug'">
  <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
  <IncludeSymbols>True</IncludeSymbols>
  <IncludeSource>True</IncludeSource>
  <SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>