Here is a minimalistic example (you should of course add your own error checking):
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <libxml/xmlmemory.h>
#include <libxml/parser.h>
#include <libxml/relaxng.h>
int
main(int argc, char *argv[])
{
int status;
xmlDoc *doc;
xmlRelaxNGPtr schema;
xmlRelaxNGValidCtxtPtr validctxt;
xmlRelaxNGParserCtxtPtr rngparser;
doc = xmlParseFile(argv[1]);
rngparser = xmlRelaxNGNewParserCtxt(argv[2]);
schema = xmlRelaxNGParse(rngparser);
validctxt = xmlRelaxNGNewValidCtxt(schema);
status = xmlRelaxNGValidateDoc(validctxt, doc);
printf("status == %d\n", status);
xmlRelaxNGFree(schema);
xmlRelaxNGFreeValidCtxt(validctxt);
xmlRelaxNGFreeParserCtxt(rngparser);
xmlFreeDoc(doc);
exit(EXIT_SUCCESS);
}
Compile this with gcc -I/usr/include/libxml2 rngval.c -o rngval -lxml2
You can check the relevant documentation at http://xmlsoft.org/html/libxml-relaxng.html