How to import .pc from a pkgconfig folder using cmake?

Question

I am currently building FFmpeg as an external project. FFmpeg builds correctly and the library is stuck inside an appropriate lib folder. And inside that lib folder a pkgconfig folder is placed inside with all of the .pc files with the information of all of the associated libraries. Of course, I could be thinking about this wrong. However when I built FFmpeg, I had to use a pkg_congig so it could link with the libraries of x264. Here is my ffmpeg_exteral.cmake file:

SET_PROPERTY(DIRECTORY PROPERTY "EP_BASE" ${ep_base})

set( FFmpeg_url "https://github.com/FFmpeg/FFmpeg.git")
set( FFmpeg_TAG "n4.2")
set(FFmpeg_depends "x264_external_download")

set(PATH_DEPENDS "${x264_LIBRARY_DIR}/pkgconfig")

ExternalProject_Add(FFmpeg_external_download
  DEPENDS ${FFmpeg_depends}
  GIT_REPOSITORY ${FFmpeg_url}
  GIT_TAG ${FFmpeg_TAG}
  UPDATE_COMMAND ""
  PATCH_COMMAND ""
  INSTALL_COMMAND ""
  INSTALL_DIR ""
  CONFIGURE_COMMAND PKG_CONFIG_PATH=${PATH_DEPENDS} <SOURCE_DIR>/configure
    --prefix=<BINARY_DIR>/build
    --enable-static
    --extra-cflags=-I${x264_INCLUDE_DIR}\ --static
    --extra-ldflags=-L${x264_LIBRARY_DIR}
    --enable-gpl
    --enable-libx264
  BUILD_COMMAND make
    -j8
)

#CACHE PATH "" seems to write the path to a file that I can set 
#library paths to. 

ExternalProject_Get_Property(FFmpeg_external_download BINARY_DIR)

set(FFmpeg_LIBRARY_DIR ${BINARY_DIR}/build/lib CACHE INTERNAL "")
set(FFmpeg_INCLUDE_DIR ${BINARY_DIR}/build/include CACHE INTERNAL "")

add_library(FFmpeg_external STATIC IMPORTED)

And this links directly with a x264_external.cmake

SET_PROPERTY(DIRECTORY PROPERTY "EP_BASE" ${ep_base})

set( x264_url "https://code.videolan.org/videolan/x264.git")
set( x264_TAG "origin/stable")


##[[
  The configure command is required because FFMPEG will try to build these 
  libraries when x264 static does not need them. 
]]
ExternalProject_Add(x264_external_download
  GIT_REPOSITORY ${x264_url}
  GIT_TAG ${x264_TAG}
  UPDATE_COMMAND ""
  PATCH_COMMAND ""
  INSTALL_COMMAND ""
  INSTALL_DIR ""
  CONFIGURE_COMMAND <SOURCE_DIR>/configure
    --prefix=<BINARY_DIR>/x264_build
    --enable-static
    --disable-opencl 
    --disable-avs 
    --disable-cli 
    --disable-ffms 
    --disable-gpac 
    --disable-lavf 
    --disable-swscale
  BUILD_COMMAND make install
    -j8
)

#CACHE PATH "" seems to write the path to a file that I can set 
#library paths to. 

ExternalProject_Get_Property(x264_external_download BINARY_DIR)

set(x264_LIBRARY_DIR ${BINARY_DIR}/x264_build/lib CACHE INTERNAL "")
set(x264_BINARY_DIR ${BINARY_DIR}/x264_build/bin CACHE INTERNAL "")
set(x264_INCLUDE_DIR ${BINARY_DIR}/x264_build/include CACHE INTERNAL "")

add_library(x264_external STATIC IMPORTED)

As you can see, FFmpeg takes the pkg_config_path from the library directory of x264 and they link together just fine. However, now that I am trying to build a simple example to make sure FFmpeg is linking with my project. I get errors of undefined references to anything from FFmpeg. This is all main.cpp has:

#include <iostream>
#include <libavcodec/avcodec.h>
#include <libavdevice/avdevice.h>
#include <libavfilter/avfilter.h>
#include <libavformat/avformat.h>
#include <libavformat/avio.h>
#include <libavutil/avutil.h>

int main()
{
  std::cout << "It compiled." << std::endl;

  avformat_network_init();

  std::cout << "It worked." << std::endl;

  avformat_network_deinit();

}

And in the cmake file that uses it, all it has is:

set(CMAKE_PREFIX_PATH ${FFmpeg_LIBRARY_DIR}/pkgconfig)

include_directories(${FFmpeg_INCLUDE_DIR})
link_directories(${FFmpeg_LIBRARY_DIR})

add_executable(Main main.cpp)

add_dependencies(Main FFmpeg_external_download)

I read that CMAKE_PREFIX_PATH will find and read in .pc files, however it doesn't seem to be doing much here, and as far as I can tell, I am not really sure how to set PkgConfig to find a custom path for a package. And maybe this is linking correctly because I am doing this wrong. Any help on how to get these to link together would be tremendous help.

EDIT: I forgot to say that FFmpeg builds 8 libraries that I am trying to link to in the lib folder. I have tried creating a library for each of them, setting their target properties to their libraries, and linking them to main using target_link_libraries and that hasn't seemed to work either.

EDIT 2: Error message

CMakeFiles/Main.dir/main.cpp.o: In function `main':
main.cpp:(.text+0x2d): undefined reference to `avformat_network_init()'
main.cpp:(.text+0x5a): undefined reference to `avformat_network_deinit()'
collect2: error: ld returned 1 exit status
Source/CMakeFiles/Main.dir/build.make:91: recipe for target 'bin/Main' failed
make[2]: *** [bin/Main] Error 1
CMakeFiles/Makefile2:242: recipe for target 'Source/CMakeFiles/Main.dir/all' failed
make[1]: *** [Source/CMakeFiles/Main.dir/all] Error 2
Makefile:83: recipe for target 'all' failed
make: *** [all] Error 2

EDIT 3: Verbose output

-L/home/sailanarmo/reeee/Superbuild/Build/FFmpeg_external_download/build/lib -Wl,-rpath,/home/sailanarmo/reeee/Superbuild/Build/FFmpeg_external_download/build/lib -lpthread ../Superbuild/Build/FFmpeg_external_download/build/lib/libavcodec.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libavdevice.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libavfilter.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libavformat.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libavutil.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libpostproc.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libswresample.a ../Superbuild/Build/FFmpeg_external_download/build/lib/libswscale.a