You can plot the bars and labels separately, and then a line segment to join them. I also handled overlapping labels in a simple way.
library(ggplot2)
library(dplyr)
df <- data.frame(group = c("A","A","A",
"B","B","B"),
sector = c("x","y","z",
"x","y","z"),
value = c(10,1,
2,1,2,100))
df <- df %>% # calculate label position
group_by(group) %>% # for each bar
arrange(desc(sector)) %>% # from bottom to top
mutate(
midy = cumsum(value) - value / 2, # base label position
dmidy = diff(c(0, midy)), # gaps between labels (and axis)
dmidy2 = pmax(dmidy, 10), # apply minimum label spacing
labely = cumsum(dmidy2) # final label position
) %>% #
arrange(group, sector) # useful for debugging
ggplot(data = df) +
geom_col(mapping = aes(x = group, y = value, fill = sector), width = 0.5) + # plot the bar
geom_text(mapping = aes(x = group, y = labely, label = value), nudge_x = 0.5) + # plot the label
geom_segment(mapping = aes(x = group, y = midy, xend = as.integer(group) + 0.4, yend = labely)) # plot the line

Created on 2020-07-06 by the reprex package (v0.3.0)