setup:
  - skip:
      features: headers
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.put_job:
        job_id: job-stats-test
        body:  >
          {
            "job_id":"job-stats-test",
            "description":"Analysis of response time by airline",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "10mb"
            },
            "data_description" : {
                "format":"xcontent",
                "time_field":"time",
                "time_format":"epoch"
            }
          }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.open_job:
        job_id: job-stats-test

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.put_job:
        job_id: jobs-get-stats-datafeed-job
        body:  >
          {
            "job_id":"jobs-get-stats-datafeed-job",
            "description":"A job with a datafeed",
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "10mb"
            },
            "data_description" : {
                "format" : "xcontent",
                "time_field":"time",
                "time_format":"yyyy-MM-dd'T'HH:mm:ssX"
            }
          }
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.open_job:
        job_id: jobs-get-stats-datafeed-job

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      ml.put_datafeed:
        datafeed_id: datafeed-1
        body:  >
          {
            "job_id":"jobs-get-stats-datafeed-job",
            "indexes":["farequote"]
          }

---
"Test get job stats after uploading data prompting the creation of some stats":

  - skip:
      features:
        - "warnings"

  - do:
      warnings:
        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.post_data:
        job_id: job-stats-test
        body: >
          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
          {"airline":"JZA","responsetime":"244.1276","time":"1403485200"}

  - do:
      warnings:
        - 'Forcing any buffered data to be processed is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.flush_job:
        job_id: job-stats-test
  - match: { flushed: true }


  - do:
      ml.get_job_stats:
        job_id: job-stats-test
  - match:  { jobs.0.job_id : job-stats-test }
  - match:  { jobs.0.data_counts.processed_record_count: 3 }
  - match:  { jobs.0.data_counts.processed_field_count: 6 }
  - match:  { jobs.0.data_counts.input_field_count: 6 }
  - gte:    { jobs.0.model_size_stats.model_bytes: 0 }
  - match:  { jobs.0.state: opened }
  - is_true:  jobs.0.node.name
  - is_true:  jobs.0.node.transport_address
  - is_true:  jobs.0.node.attributes.ml\.machine_memory
  - is_true:  jobs.0.open_time
  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
  - match:  { jobs.0.timing_stats.bucket_count: 1 }  # Records are 1h apart and bucket span is 1h so 1 bucket is produced
  - gte:    { jobs.0.timing_stats.total_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }

---
"Test get job stats for closed job":

  - skip:
      features:
        - "warnings"

  - do:
      warnings:
        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.post_data:
        job_id: job-stats-test
        body: >
          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}
          {"airline":"JZA","responsetime":"244.1276","time":"1403485200"}

  - do:
      warnings:
        - 'Forcing any buffered data to be processed is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.flush_job:
        job_id: job-stats-test
  - match: { flushed: true }

  - do:
      ml.close_job:
        job_id: job-stats-test
  - match: { closed: true }

  - do:
      ml.get_job_stats:
        job_id: job-stats-test
  - match:  { jobs.0.job_id : job-stats-test }
  - match:  { jobs.0.data_counts.processed_record_count: 3 }
  - match:  { jobs.0.data_counts.processed_field_count: 6 }
  - match:  { jobs.0.data_counts.input_field_count: 6 }
  - gt:     { jobs.0.model_size_stats.model_bytes: 0 }
  - match:  { jobs.0.state: closed }
  - is_false: jobs.0.node
  - is_false: jobs.0.open_time
  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
  - match:  { jobs.0.timing_stats.bucket_count: 1 }  # Records are 1h apart and bucket span is 1h so 1 bucket is produced
  - gte:    { jobs.0.timing_stats.total_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.minimum_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.maximum_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.average_bucket_processing_time_ms: 0.0 }
  - gte:    { jobs.0.timing_stats.exponential_average_bucket_processing_time_ms: 0.0 }

---
"Test get job stats of datafeed job that has not received any data":

  - do:
      ml.get_job_stats:
        job_id: jobs-get-stats-datafeed-job
  - match:  { jobs.0.job_id : jobs-get-stats-datafeed-job }
  - match:  { jobs.0.data_counts.processed_record_count: 0 }
  - match:  { jobs.0.model_size_stats.model_bytes : 0 }
  - match:  { jobs.0.state: opened }
  - is_true:  jobs.0.open_time
  - match:  { jobs.0.timing_stats.job_id: jobs-get-stats-datafeed-job }
  - match:  { jobs.0.timing_stats.bucket_count: 0 }
  - match:  { jobs.0.timing_stats.total_bucket_processing_time_ms: 0.0 }
  - is_false: jobs.0.timing_stats.minimum_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.maximum_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.average_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.exponential_average_bucket_processing_time_ms

---
"Test get all job stats with _all":

  - do:
      ml.get_job_stats:
        job_id: _all
  - match: { count: 2 }
  - match: { jobs.0.job_id : job-stats-test }
  - match: { jobs.0.state: opened }
  - match: { jobs.1.job_id : jobs-get-stats-datafeed-job }
  - match: { jobs.1.state: opened }

---
"Test get all job stats with wildcard":

  - do:
      ml.get_job_stats:
        job_id: "*"
  - match: { count: 2 }
  - match: { jobs.0.job_id : job-stats-test }
  - match: { jobs.0.state: opened }
  - match: { jobs.1.job_id : jobs-get-stats-datafeed-job }
  - match: { jobs.1.state: opened }

---
"Test get all job stats implicitly":

  - do:
      ml.get_job_stats: {}
  - match: { count: 2 }

---
"Test get job stats given missing job":

  - do:
      catch: missing
      ml.get_job_stats:
        job_id: unknown-job

---
"Test get job stats given pattern and allow_no_match":
  - skip:
      features:
        - "warnings"

  - do:
      ml.get_job_stats:
        job_id: "missing-*"
        allow_no_match: true
  - match: { count: 0 }

---
"Test get job stats given pattern and not allow_no_match":
  - skip:
      features:
        - "warnings"

  - do:
      catch: missing
      ml.get_job_stats:
        job_id: "missing-*"
        allow_no_match: false

---
"Test reading v54 data counts and model size stats":

  - do:
      ml.put_job:
        job_id: job-stats-v54-bwc-test
        body:  >
          {
            "analysis_config" : {
                "bucket_span": "1h",
                "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
            },
            "analysis_limits" : {
                "model_memory_limit": "10mb"
            },
            "data_description" : {
            }
          }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      indices.refresh:
        index: ".ml-anomalies*"

  # This is testing that the documents with v5.4 IDs are fetched.
  # Ideally we would use the v5.4 type but we can't put a mapping
  # for another type into the single type indices. Type isn't used
  # in the query so the test is valid
  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
        Content-Type: application/json
      index:
        index: .ml-anomalies-shared
        id: job-stats-v54-bwc-test-data-counts
        body:
          {
            job_id : job-stats-v54-bwc-test,
            processed_record_count : 10,
            processed_field_count : 0,
            input_bytes : 0,
            input_field_count : 0,
            invalid_date_count : 0,
            missing_field_count : 0,
            out_of_order_timestamp_count : 0,
            empty_bucket_count : 0,
            sparse_bucket_count : 0,
            bucket_count : 0,
            input_record_count : 0,
            latest_record_timestamp: 2000000000000
          }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
        Content-Type: application/json
      index:
        index: .ml-anomalies-shared
        id: job-stats-v54-bwc-test-model_size_stats
        body:
          {
            job_id : job-stats-v54-bwc-test,
            result_type : model_size_stats,
            model_bytes : 0,
            total_by_field_count : 101,
            total_over_field_count : 0,
            total_partition_field_count : 0,
            bucket_allocation_failures_count : 0,
            memory_status : ok,
            categorized_doc_count : 0,
            total_category_count : 0,
            frequent_category_count : 0,
            rare_category_count : 0,
            dead_category_count : 0,
            failed_category_count : 0,
            categorization_status : ok,
            log_time : 1495808248662
          }

  - do:
      headers:
        Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser
      indices.refresh:
        index: [.ml-anomalies-shared]

  - do:
      ml.get_job_stats:
        job_id: job-stats-v54-bwc-test
  - match: { jobs.0.job_id : job-stats-v54-bwc-test }
  - match: { jobs.0.data_counts.processed_record_count: 10 }
  - match: { jobs.0.model_size_stats.total_by_field_count: 101 }

---
"Test no exception on get job stats with missing index":

  - skip:
      features:
        - "warnings"

  - do:
      warnings:
        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.post_data:
        job_id: job-stats-test
        body: >
          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}

  - do:
      ml.close_job:
        job_id: jobs-get-stats-datafeed-job
  - match: { closed: true }

  - do:
      ml.close_job:
        job_id: job-stats-test
  - match: { closed: true }

  - do:
      indices.delete:
        index: .ml-anomalies-shared

  - do:
      ml.get_job_stats: {}
  - match:  { count: 2 }
  - match:  { jobs.0.job_id : job-stats-test }
  - match:  { jobs.0.data_counts.processed_record_count: 0 }
  - match:  { jobs.0.data_counts.processed_field_count: 0 }
  - match:  { jobs.0.data_counts.input_field_count: 0 }
  - match:  { jobs.0.model_size_stats.model_bytes: 0 }
  - match:  { jobs.0.state: closed }
  - is_false: jobs.0.node
  - is_false: jobs.0.open_time
  - match:  { jobs.0.timing_stats.job_id: job-stats-test }
  - match:  { jobs.0.timing_stats.bucket_count: 0 }
  - match:  { jobs.0.timing_stats.total_bucket_processing_time_ms: 0.0 }
  - is_false: jobs.0.timing_stats.minimum_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.maximum_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.average_bucket_processing_time_ms
  - is_false: jobs.0.timing_stats.exponential_average_bucket_processing_time_ms
  - match:  { jobs.1.job_id : jobs-get-stats-datafeed-job }
  - match:  { jobs.1.data_counts.processed_record_count: 0 }
  - match:  { jobs.1.data_counts.processed_field_count: 0 }
  - match:  { jobs.1.data_counts.input_field_count: 0 }
  - match:  { jobs.1.model_size_stats.model_bytes: 0 }
  - match:  { jobs.1.state: closed }
  - is_false: jobs.1.node
  - is_false: jobs.1.open_time
  - match:  { jobs.1.timing_stats.job_id: jobs-get-stats-datafeed-job }
  - match:  { jobs.1.timing_stats.bucket_count: 0 }
  - match:  { jobs.1.timing_stats.total_bucket_processing_time_ms: 0.0 }
  - is_false: jobs.1.timing_stats.minimum_bucket_processing_time_ms
  - is_false: jobs.1.timing_stats.maximum_bucket_processing_time_ms
  - is_false: jobs.1.timing_stats.average_bucket_processing_time_ms
  - is_false: jobs.1.timing_stats.exponential_average_bucket_processing_time_ms

---
"Test closed results index":

  - skip:
      features:
        - "warnings"

  - do:
      warnings:
        - 'Posting data directly to anomaly detection jobs is deprecated, in a future major version it will be compulsory to use a datafeed'
      ml.post_data:
        job_id: job-stats-test
        body: >
          {"airline":"AAL","responsetime":"132.2046","time":"1403481600"}
          {"airline":"JZA","responsetime":"990.4628","time":"1403481600"}

  - do:
      ml.close_job:
        job_id: jobs-get-stats-datafeed-job
  - match: { closed: true }

  - do:
      ml.close_job:
        job_id: job-stats-test
  - match: { closed: true }

  - do:
      ml.get_job_stats: {}
  - length: { jobs : 2 }

  - do:
      xpack.usage: {}
  - match: { ml.available: true }
  - match: { ml.enabled: true }
  - match: { ml.jobs.closed.count: 2 }

  - do:
      indices.close:
        index: .ml-anomalies-shared

  # With the index closed the low level ML API reports a problem
  - do:
      catch: /type=cluster_block_exception, reason=index \[.ml-anomalies-shared\] blocked by. \[FORBIDDEN\/.\/index closed\]/
      ml.get_job_stats: {}

  # But the high level X-Pack API returns what it can - we do this
  # so that corruption to ML doesn't blind observers of the general
  # cluster status
  - do:
      xpack.usage: {}
  - match: { ml.available: true }
  - match: { ml.enabled: true }
  - is_false: ml.jobs.closed.count
