Data <- data.table(Genotype='Test',Color=c('Ref','NAM','Core Collection'),Group=c('Elite Lines','Core Collection'),V2=c(3000,6000),TrNorm=c(1,2))
Labels <- data.table(LabColor=c('Elite Lines','Core Collection'),Label=c('Label1','Label2'),x=c(1.96,1.96),y=c(5980,5851))
ggplot(data=Data,aes(x=TrNorm,y=V2,color=Color,group=Group)) +
geom_text(data=Labels,aes(x=x,y=y,label=Label,color=LabColor),inherit.aes=F,show.legend=F) +
geom_point()
As you see, geom_text adds 'Elite Lines' to the legend, which is not supposed to be there as it is not in Data$Color.