deckard_service.proto

syntax = "proto3";

import "google/protobuf/any.proto";

package blipai.deckard;

// Golang configs
option go_package = "github.com/takenet/deckard";

// Java configs
option java_multiple_files = true;
option java_package = "ai.blip.deckard";

// C# configs
option csharp_namespace = "Takenet.Deckard";

service Deckard {
  // Adds messages to the queue. If any message already exists (same id and queue) it will be updated
  rpc Add (AddRequest) returns (AddResponse);

  // Pulls messages from the queue
  rpc Pull (PullRequest) returns (PullResponse);

  // Acks a message, marking it as successfully processed
  rpc Ack (AckRequest) returns (AckResponse);

  // Nacks a message, marking it as not processed
  rpc Nack (AckRequest) returns (AckResponse);

  // Counts number of messages on queue based on a filter request
  rpc Count (CountRequest) returns (CountResponse);

  // Removes one or many messages from the queue based on its ids
  rpc Remove (RemoveRequest) returns (RemoveResponse);

  // Discards all deckard data from storage and cache.
  // This request is only available if deckard instance is configured with MEMORY cache and storage.
  rpc Flush (FlushRequest) returns (FlushResponse);

  // Gets a message from a specific queue using its id
  // CAUTION: this should be used mainly for diagnostics and debugging purposes since it will be direct operation on the storage system
  rpc GetById (GetByIdRequest) returns (GetByIdResponse);

  // Edits a queue configuration
  rpc EditQueue (EditQueueRequest) returns (EditQueueResponse);

  // Gets the current configuration for a queue
  rpc GetQueue (GetQueueRequest) returns (GetQueueResponse);
}

/*
    Get By ID
*/

message GetByIdRequest {
  // Name of the queue to get a message
  string queue = 1;

  // The message id
  string id = 2;
}

message GetByIdResponse {
  Message message = 1;

  // A human readable string data map of the message's payload.
  //
  // This represents the payload map as a JSON string representation, useful for debugging and diagnostics
  map <string, string> human_readable_payload = 3;

  bool found = 2;
}

/*
    Remove
*/

message RemoveRequest {
  repeated string ids = 1;

  // Name of the queue to remove message
  // Provide the name of the application as a prefix using colon as the separator. Example: <application_name>:<queue_name>
  string queue = 2;
}

message RemoveResponse {
  int64 cacheRemoved = 1;

  int64 storageRemoved = 2;
}

/*
    Pull
*/

message PullRequest {
  // Full name of the queue to pull messages (including any prefixes)
  string queue = 1;

  // Number of messages to fetch.
  // Caution: as greater the amount, as more time it will take to process the request.
  // Max value is 1000 and the default value is 1
  int32 amount = 2;

  // Prefer using the max_score field instead of this one.
  // This field is deprecated and will be removed in the future.
  //  
  // The `score_filter` behaves differently than `max_score` field.
  // The `max_score` field is the upper threshold itself, but the `score_filter` will result in an upper score threshold of the current timestamp minus the score_filter value.
  //
  // Useful only when your queue's score is only based on the current timestamp to not return a message just moments after it was last used.
  // It will only return messages with score lower than the current timestamp minus the score_filter value.
  //
  // For example if your queue's score is only based on the current timestamp, this parameter will be the number of milliseconds a message must be in the queue before being returned.
  int64 score_filter = 3 [deprecated=true];

  // Sets the upper threshold for the priority score of a message to be returned in the pull request.
  //
  // Only messages with a priority score equal to or lower than the max_score value will be returned.
  //
  // The maximum score accepted by Deckard is 9007199254740992, any value higher than this will be capped to the maximum score.
  // To set this value to the minimum score accepted by Deckard, use any negative number.
  // This parameter will be ignored if set to 0 (default value).
  double max_score = 4;

  // Sets the lower threshold for the priority score required for a message to be returned.
  // Only messages with a priority score equal to or higher than the min_score value will be returned.
  // The minimum score accepted by Deckard is 0 which is also the default value
  double min_score = 5;

  // Sets the amount of time in milliseconds a message will wait be available for processing before being returned to the queue if not ACKed.
  // The default value is 300000 (5 minutes).
  int64 ack_deadline_ms = 6;
}

message PullResponse {
  // List of returned messages
  repeated Message messages = 1;
}

message Message {
  // ID is an unique identifier inside a queue.
  // Any message with the same id and queue will be considered the same message.
  string id = 1;

  // Description of the message, this should be used as a human readable string to be used in diagnostics.
  string description = 2;

  // Full name of the queue this message belongs (including any prefixes)
  string queue = 3;

  // A payload map with formatted data to be stored and used by clients.
  map <string, google.protobuf.Any> payload = 8;

  // Metadata is a map of string to be used as a key-value store.
  // It is a simple way to store data that is not part of the message payload.
  map <string, string> metadata = 4;

  // Message string payload. Is responsibility of the caller to know how to encode/decode to a useful format for its purpose.
  // This field can be used to store simple string data instead of using the payload field.
  string string_payload = 5;

  // Score represents the priority score the message currently have in the queue.
  // The lower the score, the higher the priority.
  // The maximum score accepted by Deckard is 9007199254740992 and the minimum is 0
  double score = 6;

  // Breakpoint is a field to be used as an auxiliar field for some specific use cases.
  // For example if you need to keep a record of the last result processing a message, or want to iteract with a pagination system.
  //
  // Examples: imagine a message representing a web news portal and you want to navigate through the articles. This field could be used to store the last visited article id.
  // Or imagine a message representing a user and you want to iterate through the user's publications pages. This field could be used to store the last page number you visited.
  string breakpoint = 7;

  // Diagnostics is a field holding information about the message's usage.
  // It is useful to track how many times a message was ACKed or NACKed.
  MessageDiagnostics diagnostics = 9;
}

message MessageDiagnostics {
  // Track total number of ACKs
  int64 acks = 1;

  // Track total number of NACKs
  int64 nacks = 2;

  // Track number of consecutive ACKs
  // This field will reset to 0 when a NACK is received
  int64 consecutive_acks = 3;

  // Track number of consecutive NACKs
  // This field will reset to 0 when an ACK is received
  int64 consecutive_nacks = 4;
}

message CountRequest {
  // Deprecated SearchQueryFilter
  reserved 2;

  string queue = 1;
}

message CountResponse {
  int64 count = 1;
}

/*
    Add
*/

message AddRequest {
  // List of messages to be added
  repeated AddMessage messages = 1;
}

message AddMessage {
  // 5 - Deprecated CredentialsOption
  // 2 - Deprecated Data
  // 9 - Deprecated QueueConfigurations
  reserved 5, 2, 9;

  // Unique id of this message
  string id = 1;

  // A payload map with formatted data to be stored and used by clients.
  map <string, google.protobuf.Any> payload = 10;

  // Non-formatted string payload.
  // This field can be used to store simple string data instead of using the payload field.
  string string_payload = 3;

  // Metadata is a map of string to be used as a key-value store.
  // It is a simple way to store data that is not part of the message payload.
  map <string, string> metadata = 11;

  /*
    Name of the queue to add this message

    Suggestion: to better observability, provide the name of the application using colon as the separator. Example: <application_name>:<queue_name>

    You may also add a queue prefix to the queue name using two colons as the separator. Example: <application_name>:<queue_prefix>::<queue_name>
  */
  string queue = 4;

  // Indicate this message will never expire and will only be deleted from the queue if explicitly removed.
  bool timeless = 6;

  // TTL is the amount of time in minutes this message will live in the queue.
  // TTL is not a exact time, the message may live a little longer than the specified TTL.
  int64 ttl_minutes = 7;

  // Description of the message, this should be used as a human readable string to be used in diagnostics.
  string description = 8;

  // Score represents the priority score the message currently have in the queue.
  // The score is used to determine the order of the messages returned in a pull request.
  // The lower the score, the higher the priority.
  //
  // If the score is not set (or set to 0), the score will be set with the current timestamp in milliseconds at the moment of the message creation.
  //
  // The maximum score accepted by Deckard is 9007199254740992 and the minimum is 0
  // Negative scores will be converted to 0, adding the message with the lowest score (and highest priority)
  double score = 12;
}

message AddResponse {
  // Number of created messages
  int64 created_count = 1;

  // Number of updated messages
  int64 updated_count = 2;
}

/*
  Queue Configuration
*/

message EditQueueRequest {
  // Name of the queue to be updated
  // This includes all prefixes and suffixes
  string queue = 1;

  // Configuration to apply to the queue. It will always update the queue with the newer configuration.
  // Only available fields will be updated, meaning that previously configured fields will not be change unless you explicit set it.
  // If you want to change a configuration to its default value, manually set it to its default value following each field documentation.
  QueueConfiguration configuration = 2;
}

message EditQueueResponse {
  // Name of the queue
  string queue = 1;

  bool success = 2;
}

// The queue configuration does not change instantly and can take up to 10 minutes to complete update.
message QueueConfiguration {
  // Number of max elements the queue can have.
  // 
  // To apply a max elements to a queue, set a value greater than 0.
  // To remove the max elements from a queue, set the value to -1.
  // 0 will be always ignored and the queue will not be updated.
  // 
  // All queues are unlimited by default.
  //
  // The exclusion policy will be applied to the queue when the max elements is reached:
  //
  // Messages are excluded ordered by its TTL, where the closest to expire will be excluded first.
  // If all messages have the same TTL, the oldest message will be excluded first.
  int64 max_elements = 1;
}

message GetQueueRequest {
  // Name of the queue to be updated
  // This includes all prefixes and suffixes
  string queue = 1;
}

message GetQueueResponse {
  // Name of the queue
  string queue = 1;

  // Configuration of the queue
  QueueConfiguration configuration = 2;
}

/*
    Ack/Nack
*/

message AckRequest {
  // ID of the message
  string id = 1;

  // Queue where this message is stored
  string queue = 2;

  // Reason of this result.
  // 
  // Useful for audit, mostly on 'nack' signals.
  string reason = 5;

  // This field is deprecated and will be removed in the future. If you need to change the message score, use the 'score' field.
  // 
  // The value to subtract the score and increase final message priority.
  // For example if you want to make this message to have a higher priority you can set 10000 which will represent 10s of score benefit in the default score algorithm.
  // If you want to penalize the message you can send a negative number.
  // 
  // IMPORTANT: The message will not be locked by, in the example, 10 seconds. This field is used only to increase or decrease the message priority in the priority queue.
  // 
  // This field is used only for ack requests (since in nack requests the message will return with the lowest score to the queue).
  // It will be ignored if used at the same time of 'score' or 'lock_ms' fields.
  double score_subtract = 3 [deprecated = true];

  // Breakpoint is a field to be used as an auxiliar field for some specific use cases.
  // For example if you need to keep a record of the last result processing a message, or want to iteract with a pagination system.
  //
  // Examples: imagine a message representing a web news portal and you want to navigate through the articles. This field could be used to store the last visited article id.
  // Or imagine a message representing a user and you want to iterate through the user's publications pages. This field could be used to store the last page number you visited.
  string breakpoint = 4;

  // Time in milliseconds to lock a message before returning it to the queue.
  // For NACK requests the message will be locked before returning to first position in the priority queue. You can change this behavior using the 'score' field.
  //
  // For ACK requests the message will be locked before returning to last position in the priority queue. You can change this behavior using the 'score' field.
  //
  // IMPORTANT: Deckard checks for locked messages in a 1-second precision meaning the lock have a second precision and not milliseconds.
  // This field is in milliseconds because all duration units on deckard are expressed in milliseconds and the default score algorithm uses milliseconds as well.
  int64 lock_ms = 6;
  
  // Whether the message should be removed when acked/nacked
  bool removeMessage = 7;

  // Sets the score of the message when ACKed, to override the default score algorithm.
  //
  // If used at the same time with the 'lock_ms' attribute, the message will be locked for the specified time and then returned to the queue with the specified score.
  //
  // For ACK requests, if the score is not provided (or set to 0), the message will return to the queue with the default score algorithm which is the current timestamp in milliseconds.
  //
  // For NACKs requests, if the score is not provided (or set to 0), the message will return to the queue with the minimum score accepted by Deckard which is 0.
  //
  // Negative values will be converted to 0, which is how to set the highest priority to a message in a ACK/NACK request.
  //
  // REMEMBER: the maximum score accepted by Deckard is 9007199254740992 and the minimum is 0, so values outside this range will be capped.
  double score = 10;
}

message AckResponse {
  bool success = 1;

  // The removal response if the message's removal was requested
  RemoveResponse removal_response = 2;
}

/*
    Flush
*/

message FlushRequest {}

message FlushResponse {
  bool success = 1;
}