Transform DSA and storage data

The CICS Storage dashboards requires a combination of DSA and storage data. In order to achieve this we need to add and run a transform.

Manage transformed data

The transformed data should be managed similar to our CICS Performance Analyzer data. In order to achieve this, we need to perform the following steps.

Create an index lifecycle management policy by cloning the cicspa-ilm-policy. In kibana, select Management ▶ Stack Management ▶ Data: Index Lifecycle Policies ▶ cicspa-ilm-policy ▶ Save as new policy ▶ enter transform-cicspa-dsa-storage-ilm-policy as policy name ▶ Save as new policy.
Create an index template for the transformed data.

Here is a starter index template for the transformed data that use an alias instead of a data stream for rollover:

PUT _index_template/transform-cicspa-dsa-storage
{
  "index_patterns": ["transform-cicspa-dsa-storage-*"],                 
  "template": {
    "settings": {
      "number_of_replicas": 0,
      "index.lifecycle.name": "transform-cicspa-dsa-storage-ilm-policy",      
      "index.lifecycle.rollover_alias": "transform-cicspa-dsa-storage"    
    },
    "mappings": {
      "dynamic_templates": [ {
        "strings": {
          "match_mapping_type": "string",
          "mapping": {
            "type": "keyword"
          }
        }
      } ]
    }
  }
}

Tips

You can paste API requests into the Kibana console. In Kibana, select Management ▶ Dev Tools ▶ Console.

Create the initial index for the transform-cicspa-dsa-storage alias:

PUT transform-cicspa-dsa-storage-000001
{
  "aliases": {
    "transform-cicspa-dsa-storage": {
      "is_write_index": true
    }
  }
}

Add and run the transform

Here is a starter transform:

PUT _transform/transform-cicspa-dsa-storage
{
  "frequency": "1m",
  "sync": {
    "time": {
      "field": "Collection Time",
      "delay": "60s"
    }
  },
  "source": {
    "index": [
      "cicspa_storage*",
      "cicspa_dsa*"
    ],
    "query": {
      "match_all": {}
    }
  },
  "dest": {
    "index": "transform-cicspa-dsa-storage"
  },
  "pivot": {
    "group_by": {
      "APPLID": {
        "terms": {
          "field": "APPLID"
        }
      },
      "Collection Time": {
        "terms": {
          "field": "Collection Time"
        }
      },
      "Image": {
        "terms": {
          "field": "Image"
        }
      }
    },
    "aggregations": {
            "Current EDSA Limit": {
              "max": {
                "field": "Current EDSA Limit"
              }
            },
            "Current DSA Limit": {
              "max": {
                "field": "Current DSA Limit"
              }
            },
      "CDSA": {
        "filter": {
          "term": {
            "DSA Name": "CDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "RDSA": {
        "filter": {
          "term": {
            "DSA Name": "RDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "SDSA": {
        "filter": {
          "term": {
            "DSA Name": "SDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "UDSA": {
        "filter": {
          "term": {
            "DSA Name": "UDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "ECDSA": {
        "filter": {
          "term": {
            "DSA Name": "ECDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "ERDSA": {
        "filter": {
          "term": {
            "DSA Name": "ERDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "ESDSA": {
        "filter": {
          "term": {
            "DSA Name": "ESDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "EUDSA": {
        "filter": {
          "term": {
            "DSA Name": "EUDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      },
      "ETDSA": {
        "filter": {
          "term": {
            "DSA Name": "ETDSA"
          }
        },
        "aggregations": {
          "DSASize": {
            "max": {
              "field": "Peak DSA Size"
            }
          },
          "LowestFreeStorage": {
            "max": {
              "field": "Lowest Free Storage"
            }
          }
        }
      }
    }
  }
}

In Kibana, select Management ▶ Stack Management ▶ Transforms ▶ start transform-cicspa-dsa-storage.