2

We have created a Inbound Connector and custom Management Pack.

We send data to our SCOM server using .NET SDK examples.

We can see performance data and events on SCOM console. enter image description here

But when we create Performance report (Reporting > Microsoft Generic Report Library > Performance) we do not see our Performance Counters. enter image description here

It seem we have a trouble with inserting performance data to DW database.

Please ask me about any additional information which I can provide to solve the trouble.

UPDATE

Our Management Pack source code:

<?xml version="1.0" encoding="utf-8"?><ManagementPack ContentReadable="true" SchemaVersion="2.0" OriginalSchemaVersion="1.0" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
  <Manifest>
    <Identity>
      <ID>CloudMonix.ResourceMonitoring</ID>
      <Version>1.0.0.0</Version>
    </Identity>
    <Name>CloudMonix Resource Monitoring Pack</Name>
    <References>
      <Reference Alias="System">
        <ID>System.Library</ID>
        <Version>7.5.8501.0</Version>
        <PublicKeyToken>31bf3856ad364e35</PublicKeyToken>
      </Reference>
      <Reference Alias="Health">
        <ID>System.Health.Library</ID>
        <Version>7.0.8438.6</Version>
        <PublicKeyToken>31bf3856ad364e35</PublicKeyToken>
      </Reference>
      <Reference Alias="SystemCenter">
        <ID>Microsoft.SystemCenter.Library</ID>
        <Version>7.0.8438.6</Version>
        <PublicKeyToken>31bf3856ad364e35</PublicKeyToken>
      </Reference>
    </References>
  </Manifest>
  <TypeDefinitions>
    <EntityTypes>
      <ClassTypes>
        <ClassType ID="CloudMonix.ResourceMonitoring.Resource" Accessibility="Public" Abstract="false" Base="System!System.Entity" Hosted="false" Singleton="false" Extension="false">
          <Property ID="ResourceId" Type="string" AutoIncrement="false" Key="true" CaseSensitive="false" MaxLength="256" MinLength="1" Required="false" Scale="0" />
          <Property ID="ResourceType" Type="string" AutoIncrement="false" Key="false" CaseSensitive="false" MaxLength="256" MinLength="1" Required="false" Scale="0" />
          <Property ID="ResourceGroups" Type="string" AutoIncrement="false" Key="false" CaseSensitive="false" MaxLength="1024" MinLength="1" Required="false" Scale="0" />
        </ClassType>
      </ClassTypes>
    </EntityTypes>
    <ModuleTypes>
      <WriteActionModuleType ID="CloudMonix.ResourceMonitoring.SetStateAction" Accessibility="Internal" Batching="false">
        <Configuration>
          <IncludeSchemaTypes>
            <SchemaType>Health!System.Health.AlertSchema</SchemaType>
          </IncludeSchemaTypes>
          <xsd:element name="ManagementGroupId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
          <xsd:element name="MonitorId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
          <xsd:element name="ManagedEntityId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
          <xsd:element name="HealthState" type="System.Health.AlertHealthState" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
        </Configuration>
        <ModuleImplementation Isolation="Any">
          <Native>
            <ClassID>44cbc334-8b5f-4cb6-bee0-6bdcbc80e8d5</ClassID>
          </Native>
        </ModuleImplementation>
        <InputType>System!System.BaseData</InputType>
      </WriteActionModuleType>
      <WriteActionModuleType ID="CloudMonix.ResourceMonitoring.TargetSetStateAction" Accessibility="Internal" Batching="false">
        <Configuration>
          <IncludeSchemaTypes>
            <SchemaType>Health!System.Health.AlertSchema</SchemaType>
          </IncludeSchemaTypes>
          <xsd:element name="MonitorId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
          <xsd:element name="HealthState" type="System.Health.AlertHealthState" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
        </Configuration>
        <ModuleImplementation Isolation="Any">
          <Composite>
            <MemberModules>
              <WriteAction ID="WA" TypeID="CloudMonix.ResourceMonitoring.SetStateAction">
                <ManagementGroupId>$Target/ManagementGroup/Id$</ManagementGroupId>
                <MonitorId>$Config/MonitorId$</MonitorId>
                <ManagedEntityId>$Data/ManagedEntityId$</ManagedEntityId>
                <HealthState>$Config/HealthState$</HealthState>
              </WriteAction>
            </MemberModules>
            <Composition>
              <Node ID="WA" />
            </Composition>
          </Composite>
        </ModuleImplementation>
        <InputType>System!System.BaseData</InputType>
      </WriteActionModuleType>
      <WriteActionModuleType ID="CloudMonix.ResourceMonitoring.TargetSetSuccessAction" Accessibility="Public" Batching="false">
        <Configuration>
          <xsd:element name="MonitorId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
        </Configuration>
        <ModuleImplementation Isolation="Any">
          <Composite>
            <MemberModules>
              <WriteAction ID="WA" TypeID="CloudMonix.ResourceMonitoring.TargetSetStateAction">
                <MonitorId>$Config/MonitorId$</MonitorId>
                <HealthState>Success</HealthState>
              </WriteAction>
            </MemberModules>
            <Composition>
              <Node ID="WA" />
            </Composition>
          </Composite>
        </ModuleImplementation>
        <InputType>System!System.BaseData</InputType>
      </WriteActionModuleType>
      <WriteActionModuleType ID="CloudMonix.ResourceMonitoring.TargetSetWarningAction" Accessibility="Public" Batching="false">
        <Configuration>
          <xsd:element name="MonitorId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
        </Configuration>
        <ModuleImplementation Isolation="Any">
          <Composite>
            <MemberModules>
              <WriteAction ID="WA" TypeID="CloudMonix.ResourceMonitoring.TargetSetStateAction">
                <MonitorId>$Config/MonitorId$</MonitorId>
                <HealthState>Warning</HealthState>
              </WriteAction>
            </MemberModules>
            <Composition>
              <Node ID="WA" />
            </Composition>
          </Composite>
        </ModuleImplementation>
        <InputType>System!System.BaseData</InputType>
      </WriteActionModuleType>
      <WriteActionModuleType ID="CloudMonix.ResourceMonitoring.TargetSetErrorAction" Accessibility="Public" Batching="false">
        <Configuration>
          <xsd:element name="MonitorId" type="xsd:string" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />
        </Configuration>
        <ModuleImplementation Isolation="Any">
          <Composite>
            <MemberModules>
              <WriteAction ID="WA" TypeID="CloudMonix.ResourceMonitoring.TargetSetStateAction">
                <MonitorId>$Config/MonitorId$</MonitorId>
                <HealthState>Error</HealthState>
              </WriteAction>
            </MemberModules>
            <Composition>
              <Node ID="WA" />
            </Composition>
          </Composite>
        </ModuleImplementation>
        <InputType>System!System.BaseData</InputType>
      </WriteActionModuleType>
    </ModuleTypes>
  </TypeDefinitions>
  <Monitoring>
    <Rules>
      <Rule ID="CloudMonix.ResourceMonitoring.Monitor.Success" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" ConfirmDelivery="false" Remotable="true" Priority="Normal" DiscardLevel="100">
        <Category>StateCollection</Category>
        <DataSources>
          <DataSource ID="DataSource" TypeID="SystemCenter!Microsoft.SystemCenter.SdkEventProvider" />
        </DataSources>
        <ConditionDetection ID="ConditionDetection" TypeID="System!System.ExpressionFilter">
          <Expression>
            <SimpleExpression>
              <ValueExpression>
                <XPathQuery>EventNumber</XPathQuery>
              </ValueExpression>
              <Operator>Equal</Operator>
              <ValueExpression>
                <Value>1</Value>
              </ValueExpression>
            </SimpleExpression>
          </Expression>
        </ConditionDetection>
        <WriteActions>
          <WriteAction ID="WriteAction" TypeID="CloudMonix.ResourceMonitoring.TargetSetSuccessAction">
            <MonitorId>$MPElement[Name="CloudMonix.ResourceMonitoring.Monitor"]$</MonitorId>
          </WriteAction>
        </WriteActions>
      </Rule>
      <Rule ID="CloudMonix.ResourceMonitoring.Monitor.Warning" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" ConfirmDelivery="false" Remotable="true" Priority="Normal" DiscardLevel="100">
        <Category>StateCollection</Category>
        <DataSources>
          <DataSource ID="DataSource" TypeID="SystemCenter!Microsoft.SystemCenter.SdkEventProvider" />
        </DataSources>
        <ConditionDetection ID="ConditionDetection" TypeID="System!System.ExpressionFilter">
          <Expression>
            <SimpleExpression>
              <ValueExpression>
                <XPathQuery>EventNumber</XPathQuery>
              </ValueExpression>
              <Operator>Equal</Operator>
              <ValueExpression>
                <Value>2</Value>
              </ValueExpression>
            </SimpleExpression>
          </Expression>
        </ConditionDetection>
        <WriteActions>
          <WriteAction ID="WriteAction" TypeID="CloudMonix.ResourceMonitoring.TargetSetWarningAction">
            <MonitorId>$MPElement[Name="CloudMonix.ResourceMonitoring.Monitor"]$</MonitorId>
          </WriteAction>
        </WriteActions>
      </Rule>
      <Rule ID="CloudMonix.ResourceMonitoring.Monitor.Error" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" ConfirmDelivery="false" Remotable="true" Priority="Normal" DiscardLevel="100">
        <Category>StateCollection</Category>
        <DataSources>
          <DataSource ID="DataSource" TypeID="SystemCenter!Microsoft.SystemCenter.SdkEventProvider" />
        </DataSources>
        <ConditionDetection ID="ConditionDetection" TypeID="System!System.ExpressionFilter">
          <Expression>
            <SimpleExpression>
              <ValueExpression>
                <XPathQuery>EventNumber</XPathQuery>
              </ValueExpression>
              <Operator>Equal</Operator>
              <ValueExpression>
                <Value>3</Value>
              </ValueExpression>
            </SimpleExpression>
          </Expression>
        </ConditionDetection>
        <WriteActions>
          <WriteAction ID="WriteAction" TypeID="CloudMonix.ResourceMonitoring.TargetSetErrorAction">
            <MonitorId>$MPElement[Name="CloudMonix.ResourceMonitoring.Monitor"]$</MonitorId>
          </WriteAction>
        </WriteActions>
      </Rule>
    </Rules>
    <Monitors>
      <AggregateMonitor ID="CloudMonix.ResourceMonitoring.Monitor" Accessibility="Public" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" ParentMonitorID="Health!System.Health.EntityState" Remotable="true" Priority="Normal">
        <Category>StateCollection</Category>
        <Algorithm>WorstOf</Algorithm>
      </AggregateMonitor>
    </Monitors>
  </Monitoring>
  <Presentation>
    <Views>
      <View ID="CloudMonix.ResourceMonitoring.MainView" Accessibility="Public" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" TypeID="SystemCenter!Microsoft.SystemCenter.StateViewType" Visible="true">
        <Category>Operations</Category>
        <Criteria>
          <InMaintenanceMode>false</InMaintenanceMode>
        </Criteria>
        <Presentation>
          <ColumnInfo Index="0" SortIndex="-1" Width="100" Grouped="false" Sorted="true" IsSortable="true" Visible="true" SortOrder="Ascending">
            <Name>State</Name>
            <Id>CloudMonix.ResourceMonitoring.Resource</Id>
          </ColumnInfo>
          <ColumnInfo Index="1" SortIndex="-1" Width="100" Grouped="false" Sorted="false" IsSortable="true" Visible="true" SortOrder="Ascending">
            <Name>Resource Type</Name>
            <Id>ResourceType</Id>
          </ColumnInfo>
          <ColumnInfo Index="2" SortIndex="-1" Width="100" Grouped="false" Sorted="false" IsSortable="true" Visible="true" SortOrder="Ascending">
            <Name>Resource Name</Name>
            <Id>DisplayName</Id>
          </ColumnInfo>
          <ColumnInfo Index="3" SortIndex="-1" Width="100" Grouped="false" Sorted="false" IsSortable="true" Visible="true" SortOrder="Ascending">
            <Name>Resource Groups</Name>
            <Id>ResourceGroups</Id>
          </ColumnInfo>
        </Presentation>
        <Target />
      </View>
    </Views>
    <Folders>
      <Folder ID="CloudMonix.ResourceMonitoring.MainFolder" Accessibility="Public" ParentFolder="SystemCenter!Microsoft.SystemCenter.Monitoring.ViewFolder.Root" />
    </Folders>
    <FolderItems>
      <FolderItem ElementID="CloudMonix.ResourceMonitoring.MainView" ID="CloudMonix.ResourceMonitoring.MainView" Folder="CloudMonix.ResourceMonitoring.MainFolder" />
    </FolderItems>
  </Presentation>
  <LanguagePacks>
    <LanguagePack ID="ENU" IsDefault="false">
      <DisplayStrings>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring">
          <Name>CloudMonix Resource Monitoring</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.MainFolder">
          <Name>CloudMonix Folder</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.MainView">
          <Name>CloudMonix Resource View</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.Monitor">
          <Name>CloudMonix Health State Monitor</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.Resource">
          <Name>CloudMonix Resource</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.Resource" SubElementID="ResourceId">
          <Name>Resource Id</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.Resource" SubElementID="ResourceType">
          <Name>Resource Type</Name>
        </DisplayString>
        <DisplayString ElementID="CloudMonix.ResourceMonitoring.Resource" SubElementID="ResourceGroups">
          <Name>Resource Groups</Name>
        </DisplayString>
      </DisplayStrings>
    </LanguagePack>
  </LanguagePacks>
</ManagementPack>

UPDATE 2

Our source code is very simple:

var customMonitoringPerformanceDatas = performanceDatas
    .Select(n => new CustomMonitoringPerformanceData("CloudMonix", n.MetricName, n.Value)
    {
        TimeSampled = n.TimeStamp
    })
    .OrderBy(n => n.TimeSampled)
    .ToList();

monitoringObject.InsertCustomMonitoringPerformanceData(customMonitoringPerformanceDatas);

UPDATE 3

I have implemented next Rule:

<Rule ID="CloudMonix.ResourceMonitoring.PerformanceData" Enabled="true" Target="CloudMonix.ResourceMonitoring.Resource" ConfirmDelivery="false" Remotable="true" Priority="Normal" DiscardLevel="100">
  <Category>PerformanceCollection</Category>
  <DataSources>
    <DataSource ID="DataSource" TypeID="SystemCenter!Microsoft.SystemCenter.SdkPerformanceDataProvider" />
    </DataSource>         
  </DataSources>
  <WriteActions>
    <WriteAction ID="WriteToDB" TypeID="SystemCenter!Microsoft.SystemCenter.CollectPerformanceData" />
    <WriteAction ID="WriteToDW" TypeID="SystemCenterDataWarehouse!Microsoft.SystemCenter.DataWarehouse.PublishPerformanceDataCustomChannel">
      <ChannelId>69173604-6F23-4F98-3383-F3500BA26228</ChannelId>
    </WriteAction>
  </WriteActions>
</Rule>

I used my custom Connector ID for ChannelId in the Rule. But I do not see Performance counters unfortunately.

UPDATE 4

I have investigated SCOM DB and saw my trouble. Our data were collected to DW database.

But all our data were collected with ONE RULE ID.

enter image description here

As SCOM uses ONLY ONE RULE ID for all data we can see ONLY ONE PERFORMANCE COUNTER NAME.

How we need to collect properly performance data using SDK?

Alexander I.
  • 2,380
  • 3
  • 17
  • 42
  • 1
    That's an interesting question. Normally, if you create a native SCOM performance collection rule, you shall include two write actions: one for OpsDB and another for DW DB, so looks like there is no automatic process to aggregate any perf data from Ops to DW. And your example confirms it. Let me do some research if I can find anything. Like that: – Max Jun 06 '18 at 21:27
  • Looks like there is a standard rule to automatically sync perf data from Ops to DW, but rule's descriptions mentions "standard channel". Presumable, custom perf data has non-standard channel id. Both actions are native -- so cannot decompile. – Max Jun 06 '18 at 23:14
  • @Max Thank you for your help again! Please review our Management Pack source code. We do not use any additional rules to collect data to OpsDB or DW DB. If I understand you correctly, wee need add additional rules. I'm right? – Alexander I. Jun 07 '18 at 09:29
  • Seems that you need to "publish" your data to DW or "enable" it for DW sync. But I'm not able to figure out "how" so far. (( Would you, also, mind to share your c# code? I understand, it's very easy and just a like-to-like of the example code, but it may help if I see it. – Max Jun 07 '18 at 21:50
  • @Max I have added Source Code to the question. I think we need implement some changes in Management Pack... SDK has no any documentation about DW database. – Alexander I. Jun 08 '18 at 09:41
  • Aha! Is your date/time in UTC? it (I believe) should have value in UTC and DataKind property set to Universal Time as well. – Max Jun 08 '18 at 10:23
  • Yes, It is UTC. But I will set DataKind again and check... I will inform you about test result soon. – Alexander I. Jun 08 '18 at 10:38
  • @Max I have checked TimeSampled and it was UTC. I have specify KInd again but there no results unfortunately. I think we need add additional Rules as you described in your first comment. But I do not know how implement these Rules. Maybe you have any example or link how to do it? – Alexander I. Jun 08 '18 at 11:28
  • Maybe it is correct [link](https://msdn.microsoft.com/en-us/library/ff472335.aspx) for our task? – Alexander I. Jun 08 '18 at 11:31
  • Seems I'm running out of ideas. Yes, I thought about the publishing task, but there is no place to add it, because there is no perf data collection rule. – Max Jun 10 '18 at 21:51
  • Hi @Max please review next [link](https://social.technet.microsoft.com/wiki/contents/articles/18086.scom-2012-collecting-unixlinux-data-with-performance-collection-rules.aspx). I think we can add Rule as described in the article. But I do not understand how collect all performance data to DW. We have performance data with multiple names. Could you assist please? – Alexander I. Jun 14 '18 at 13:04
  • Sorry, a bit busy this week. – Max Jun 18 '18 at 00:13
  • Hi @Max. We have found solution. Please review my [answer](https://stackoverflow.com/questions/50721626/how-to-insert-performance-data-to-dw-database-using-scom-sdk/51018911#51018911). I think it will be interesting for you... Please upvote my answer also. – Alexander I. Jun 25 '18 at 08:17

1 Answers1

3

We have fixed our trouble!

I have reviewed SCOM DW database and saw that we need to implement separate Rule for each metric.

We can find similar issue and description on Michel Kamp blog.

We used SCOM SDK to create dynamic Rules and add these Rules to our Management Pack.

Detailed code example is here:

const string conditionName = "System.ExpressionFilter";
const string dataSourceName = "Microsoft.SystemCenter.SdkPerformanceDataProvider";
const string writeActionName = "Microsoft.SystemCenter.DataWarehouse.PublishPerformanceData";

var managementClass = GetManagementPackClass(managementGroup, ManagementPackClass);

var criteria = new ManagementPackModuleTypeCriteria($"Name = '{conditionName}' OR Name = '{dataSourceName}' OR Name = '{writeActionName}'");
var moduleTypes = managementGroup.Monitoring.GetModuleTypes(criteria).ToList();

var conditionType = moduleTypes.Single(n => n.Name == conditionName);
var dataSourceType = moduleTypes.Single(n => n.Name == dataSourceName);
var writeActionType = moduleTypes.Single(n => n.Name == writeActionName);

foreach (var ruleName in missingRuleNames)
{
    var counterName = ruleName.Replace(MonitoringRulePrefix, string.Empty);

    var rule = new ManagementPackRule(managementPack, ruleName)
    {
        Target = managementClass,
        Category = ManagementPackCategoryType.PerformanceCollection,
        Enabled = ManagementPackMonitoringLevel.@true,
        ConfirmDelivery = false
    };

    rule.ConditionDetection = new ManagementPackConditionDetectionModule(rule, "ConditionDetection")
    {
        TypeID = (ManagementPackConditionDetectionModuleType) conditionType,
        Configuration = $@"
            <Expression>
                <SimpleExpression>
                    <ValueExpression>
                        <XPathQuery>CounterName</XPathQuery>
                    </ValueExpression>
                    <Operator>Equal</Operator>
                    <ValueExpression>
                        <Value>{counterName}</Value>
                    </ValueExpression>
                </SimpleExpression>
            </Expression>"
    };

    var dataSource = new ManagementPackDataSourceModule(rule, "DataSource")
    {
        TypeID = (ManagementPackDataSourceModuleType) dataSourceType
    };
    rule.DataSourceCollection.Add(dataSource);

    var action = new ManagementPackWriteActionModule(rule, "WriteToDataWarehouse")
    {
        TypeID = (ManagementPackWriteActionModuleType) writeActionType
    };
    rule.WriteActionCollection.Add(action);
}

managementPack.Verify();
managementPack.AcceptChanges();
Alexander I.
  • 2,380
  • 3
  • 17
  • 42
  • Oh, yes! That's really cool. Now you can create management packs programmatically on fly. But I'm a bit confused. Do you still insert data via SDK and a connector? So, just inserting data is not enough, you also need a "receiving rule" with the special data source module Microsoft.SystemCenter.SdkPerformanceDataProvider, right? That's a really cool finding indeed! Regarding the article, yes, I knew about it, but initially it was not a case, because there were no such "receiving rule". Is that right? – Max Jun 26 '18 at 09:08
  • Saw you update. ChannelId is the key. Right. You must write a blog article! – Max Jun 26 '18 at 09:11