Yeah... something really weird going on there.
My guess would be that UIKit has some sort of algorithm that is trying to apply "readability" to the buttons. Clearly, though, that's not the goal here.
After some quick testing, one way to get around this is to add the button as a subview to a clear "container" view, and then add that container view to the stack view.
Some sample code:
class SampleViewController: UIViewController {
let filterStackViewA: UIStackView = {
let v = UIStackView()
v.spacing = 8
v.distribution = .fill
v.alignment = .center
return v
}()
let filterStackViewB: UIStackView = {
let v = UIStackView()
v.spacing = 8
v.distribution = .fill
v.alignment = .center
return v
}()
let scrollViewA: UIScrollView = {
let v = UIScrollView()
v.backgroundColor = .systemYellow
return v
}()
let scrollViewB: UIScrollView = {
let v = UIScrollView()
v.backgroundColor = .systemOrange
return v
}()
override func viewDidLoad() {
super.viewDidLoad()
view.backgroundColor = .darkGray
let addBtn = UIButton()
addBtn.setTitle("Add Button", for: [])
addBtn.setTitleColor(.white, for: .normal)
addBtn.setTitleColor(.lightGray, for: .highlighted)
addBtn.backgroundColor = .systemGreen
addBtn.layer.cornerRadius = 8
let labelA: UILabel = {
let v = UILabel()
v.textColor = .white
v.font = .systemFont(ofSize: 15.0, weight: .light)
v.text = "Buttons added to \"Container\" views..."
return v
}()
let labelB: UILabel = {
let v = UILabel()
v.textColor = .white
v.font = .systemFont(ofSize: 15.0, weight: .light)
v.text = "Buttons added directly to stack view..."
return v
}()
[addBtn, scrollViewA, scrollViewB, filterStackViewA, filterStackViewB, labelA, labelB].forEach { v in
v.translatesAutoresizingMaskIntoConstraints = false
}
view.addSubview(addBtn)
view.addSubview(labelA)
view.addSubview(scrollViewA)
scrollViewA.addSubview(filterStackViewA)
view.addSubview(labelB)
view.addSubview(scrollViewB)
scrollViewB.addSubview(filterStackViewB)
let g = view.safeAreaLayoutGuide
let cgA = scrollViewA.contentLayoutGuide
let fgA = scrollViewA.frameLayoutGuide
let cgB = scrollViewB.contentLayoutGuide
let fgB = scrollViewB.frameLayoutGuide
NSLayoutConstraint.activate([
addBtn.topAnchor.constraint(equalTo: g.topAnchor, constant: 40.0),
addBtn.widthAnchor.constraint(equalToConstant: 200.0),
addBtn.centerXAnchor.constraint(equalTo: g.centerXAnchor),
labelA.topAnchor.constraint(equalTo: addBtn.bottomAnchor, constant: 40.0),
labelA.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
labelA.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
scrollViewA.topAnchor.constraint(equalTo: labelA.bottomAnchor, constant: 8.0),
scrollViewA.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
scrollViewA.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
scrollViewA.heightAnchor.constraint(equalToConstant: 60.0),
filterStackViewA.topAnchor.constraint(equalTo: cgA.topAnchor, constant: 4.0),
filterStackViewA.leadingAnchor.constraint(equalTo: cgA.leadingAnchor, constant: 8.0),
filterStackViewA.trailingAnchor.constraint(equalTo: cgA.trailingAnchor, constant: -8.0),
filterStackViewA.bottomAnchor.constraint(equalTo: cgA.bottomAnchor, constant: -4.0),
filterStackViewA.heightAnchor.constraint(equalTo: fgA.heightAnchor, constant: -8.0),
labelB.topAnchor.constraint(equalTo: filterStackViewA.bottomAnchor, constant: 20.0),
labelB.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
labelB.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
scrollViewB.topAnchor.constraint(equalTo: labelB.bottomAnchor, constant: 8.0),
scrollViewB.leadingAnchor.constraint(equalTo: g.leadingAnchor, constant: 20.0),
scrollViewB.trailingAnchor.constraint(equalTo: g.trailingAnchor, constant: -20.0),
scrollViewB.heightAnchor.constraint(equalToConstant: 60.0),
filterStackViewB.topAnchor.constraint(equalTo: cgB.topAnchor, constant: 4.0),
filterStackViewB.leadingAnchor.constraint(equalTo: cgB.leadingAnchor, constant: 8.0),
filterStackViewB.trailingAnchor.constraint(equalTo: cgB.trailingAnchor, constant: -8.0),
filterStackViewB.bottomAnchor.constraint(equalTo: cgB.bottomAnchor, constant: -4.0),
filterStackViewB.heightAnchor.constraint(equalTo: fgB.heightAnchor, constant: -8.0),
])
addBtn.addTarget(self, action: #selector(addButton), for: .touchUpInside)
}
@objc func addButton() {
// unwrap optional system image
guard let img = UIImage(systemName: "xmark.circle", withConfiguration: UIImage.SymbolConfiguration(scale: .unspecified))
else { return }
var newButtonConfiguration = UIButton.Configuration.filled()
newButtonConfiguration.title = randomString(length: Int.random(in: 5..<15))
newButtonConfiguration.image = img
// let's add a new button to a "container" view
let newButtonA = UIButton(configuration: newButtonConfiguration)
newButtonA.translatesAutoresizingMaskIntoConstraints = false
let v = UIView()
v.addSubview(newButtonA)
NSLayoutConstraint.activate([
newButtonA.topAnchor.constraint(equalTo: v.topAnchor),
newButtonA.leadingAnchor.constraint(equalTo: v.leadingAnchor),
newButtonA.trailingAnchor.constraint(equalTo: v.trailingAnchor),
newButtonA.bottomAnchor.constraint(equalTo: v.bottomAnchor),
])
// add the container view to the stack view
filterStackViewA.addArrangedSubview(v)
// let's add a new button directly to the stack view
let newButtonB = UIButton(configuration: newButtonConfiguration)
filterStackViewB.addArrangedSubview(newButtonB)
newButtonA.addTarget(self, action: #selector(removeMe(_:)), for: .touchUpInside)
newButtonB.addTarget(self, action: #selector(removeMe(_:)), for: .touchUpInside)
// let's make sure the new button is visible
// note: this is just for example...
// if we rapidly tap and add buttons, this can easily "miss" the last button
// so don't expect this to be "production" code
DispatchQueue.main.async {
let sz = self.scrollViewA.contentSize
let rA = CGRect(x: sz.width - 1.0, y: 0.0, width: 1.0, height: 1.0)
self.scrollViewA.scrollRectToVisible(rA, animated: true)
let szB = self.scrollViewB.contentSize
let rB = CGRect(x: szB.width - 1.0, y: 0.0, width: 1.0, height: 1.0)
self.scrollViewB.scrollRectToVisible(rB, animated: true)
}
}
@objc func removeMe(_ sender: UIButton) {
// get a reference to the tapped button's superview
guard let sv = sender.superview else { return }
var v: UIView!
if sv is UIStackView {
// we tapped a button added directly to the stack view
v = sender
} else {
// we tapped a button that's in a "container" view
v = sv
}
guard let st = v.superview as? UIStackView,
let idx = st.arrangedSubviews.firstIndex(of: v),
filterStackViewA.arrangedSubviews.count > idx,
filterStackViewB.arrangedSubviews.count > idx
else {
print("something's not setup right, so return")
return
}
let bA = filterStackViewA.arrangedSubviews[idx]
let bB = filterStackViewB.arrangedSubviews[idx]
// let's animate the buttons away
bA.isHidden = true
bB.isHidden = true
UIView.animate(withDuration: 0.5, animations: {
self.view.layoutIfNeeded()
}, completion: { _ in
bA.removeFromSuperview()
bB.removeFromSuperview()
})
}
func randomString(length: Int) -> String {
let letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
return String((0..<length).map{ _ in letters.randomElement()! })
}
}
It will look like this when running:

Each tap of "Add Button" will add a new button to each scrolling-stackView, with the top set using buttons in container views. Tapping any of those buttons will remove it from its stack view, along with its "twin" from the other one.