setFilter not responding in DataProvider for Grid

I am trying to filter a grid simply with a String picked up from a filter textField, but the grid does not update when filtering. I have checked that the backend fetch calls gives filtered outputs, but the issue seems to be with updating of the loaded data.

This is the dataprovider

    ConfigurableFilterDataProvider<Contact, Void, String> getDataProvider() {
        ContactService service = services.getContactService();
        DataProvider<Contact, String> dataProvider =
                DataProvider.fromFilteringCallbacks(query -> {
                    // getFilter returns Optional<String>
                    String filter = query.getFilter()
                            .orElse(null);
                    return service.fetchContacts(loggedInUser.getContactRef().getClientRef(), filter,
                            query.getLimit(), query.getOffset(), null).stream();
                }, query -> {
                    String filter = query.getFilter()
                            .orElse(null);
                    return service.countContacts(loggedInUser.getContactRef().getClientRef(), filter);
                });
        return dataProvider.withConfigurableFilter();
    }

And the simple textfield that serves as input for the filter text

        nameFilter.addValueChangeListener(event -> {
            String filter = event.getValue();
            if (StringUtils.isBlank(filter)) {
                filter = null;
            }
            dataProvider.setFilter(filter);
            dataProvider.refreshAll();
        });

The backend call looks like this

    public List<Contact> fetchContacts(Client client, String filter, int limit, int offset, List<QuerySortOrder> sortOrders) {
        if (filter == null) {
            filter = "";
        }
        filter = "%" + filter.toLowerCase().trim() + "%";
        if (sortOrders == null || sortOrders.isEmpty()) {
            sortOrders = new ArrayList<>();
            sortOrders.add(new QuerySortOrder(SORT_ON_NAME, SortDirection.ASCENDING));
            sortOrders.add(new QuerySortOrder(SORT_ON_LASTNAME, SortDirection.ASCENDING));
        }

        List<Contact> employees = contactDao.fetchContactsFromQuerySortOrder(client, filter, limit, offset, sortOrders);

        return employees;
    }

    public int countContacts(Client client, String filter) {
        return fetchContacts(client, filter, 100000, 0, null).size();
    }

Have also tried using the below ContactDataProvider().withConfigurableFilter() but the setFilter() still does not filter the grid.

public class ContactDataProvider extends AbstractBackEndDataProvider<Contact, String> {
    private final User loggedInUser;
    private final ContactService contactService;

    public ContactDataProvider(ContactService contactService) {
        this.contactService = contactService;
        this.loggedInUser = SecurityUtils.getUser();
    }

    @Override
    protected Stream<Contact> fetchFromBackEnd(Query<Contact, String> query) {
        return contactService.fetchContacts(loggedInUser.getContactRef().getClientRef(), query.getFilter().orElse(null), query.getLimit(), query.getOffset(), query.getSortOrders()).stream();
    }

    @Override
    protected int sizeInBackEnd(Query<Contact, String> query) {
        return contactService.countContacts(loggedInUser.getContactRef().getClientRef(), query.getFilter().orElse(null));
    }

    @Override
    public Object getId(Contact contact) {
        return contact.getId();
    }
}